1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ 2 3 /* Authors: Cheng Xu <[email protected]> */ 4 /* Kai Shen <[email protected]> */ 5 /* Copyright (c) 2020-2022, Alibaba Group. */ 6 7 #ifndef __ERDMA_HW_H__ 8 #define __ERDMA_HW_H__ 9 10 #include <linux/kernel.h> 11 #include <linux/types.h> 12 #include <linux/if_ether.h> 13 14 /* PCIe device related definition. */ 15 #define ERDMA_PCI_WIDTH 64 16 #define ERDMA_FUNC_BAR 0 17 #define ERDMA_MISX_BAR 2 18 19 #define ERDMA_BAR_MASK (BIT(ERDMA_FUNC_BAR) | BIT(ERDMA_MISX_BAR)) 20 21 /* MSI-X related. */ 22 #define ERDMA_NUM_MSIX_VEC 32U 23 #define ERDMA_MSIX_VECTOR_CMDQ 0 24 25 /* RoCEv2 related */ 26 #define ERDMA_ROCEV2_GID_SIZE 16 27 #define ERDMA_MAX_PKEYS 1 28 #define ERDMA_DEFAULT_PKEY 0xFFFF 29 30 /* erdma device protocol type */ 31 enum erdma_proto_type { 32 ERDMA_PROTO_IWARP = 0, 33 ERDMA_PROTO_ROCEV2 = 1, 34 ERDMA_PROTO_COUNT = 2, 35 }; 36 37 /* PCIe Bar0 Registers. */ 38 #define ERDMA_REGS_VERSION_REG 0x0 39 #define ERDMA_REGS_DEV_PROTO_REG 0xC 40 #define ERDMA_REGS_DEV_CTRL_REG 0x10 41 #define ERDMA_REGS_DEV_ST_REG 0x14 42 #define ERDMA_REGS_NETDEV_MAC_L_REG 0x18 43 #define ERDMA_REGS_NETDEV_MAC_H_REG 0x1C 44 #define ERDMA_REGS_CMDQ_SQ_ADDR_L_REG 0x20 45 #define ERDMA_REGS_CMDQ_SQ_ADDR_H_REG 0x24 46 #define ERDMA_REGS_CMDQ_CQ_ADDR_L_REG 0x28 47 #define ERDMA_REGS_CMDQ_CQ_ADDR_H_REG 0x2C 48 #define ERDMA_REGS_CMDQ_DEPTH_REG 0x30 49 #define ERDMA_REGS_CMDQ_EQ_DEPTH_REG 0x34 50 #define ERDMA_REGS_CMDQ_EQ_ADDR_L_REG 0x38 51 #define ERDMA_REGS_CMDQ_EQ_ADDR_H_REG 0x3C 52 #define ERDMA_REGS_AEQ_ADDR_L_REG 0x40 53 #define ERDMA_REGS_AEQ_ADDR_H_REG 0x44 54 #define ERDMA_REGS_AEQ_DEPTH_REG 0x48 55 #define ERDMA_REGS_GRP_NUM_REG 0x4c 56 #define ERDMA_REGS_AEQ_DB_REG 0x50 57 #define ERDMA_CMDQ_SQ_DB_HOST_ADDR_REG 0x60 58 #define ERDMA_CMDQ_CQ_DB_HOST_ADDR_REG 0x68 59 #define ERDMA_CMDQ_EQ_DB_HOST_ADDR_REG 0x70 60 #define ERDMA_AEQ_DB_HOST_ADDR_REG 0x78 61 #define ERDMA_REGS_STATS_TSO_IN_PKTS_REG 0x80 62 #define ERDMA_REGS_STATS_TSO_OUT_PKTS_REG 0x88 63 #define ERDMA_REGS_STATS_TSO_OUT_BYTES_REG 0x90 64 #define ERDMA_REGS_STATS_TX_DROP_PKTS_REG 0x98 65 #define ERDMA_REGS_STATS_TX_BPS_METER_DROP_PKTS_REG 0xa0 66 #define ERDMA_REGS_STATS_TX_PPS_METER_DROP_PKTS_REG 0xa8 67 #define ERDMA_REGS_STATS_RX_PKTS_REG 0xc0 68 #define ERDMA_REGS_STATS_RX_BYTES_REG 0xc8 69 #define ERDMA_REGS_STATS_RX_DROP_PKTS_REG 0xd0 70 #define ERDMA_REGS_STATS_RX_BPS_METER_DROP_PKTS_REG 0xd8 71 #define ERDMA_REGS_STATS_RX_PPS_METER_DROP_PKTS_REG 0xe0 72 #define ERDMA_REGS_CEQ_DB_BASE_REG 0x100 73 #define ERDMA_CMDQ_SQDB_REG 0x200 74 #define ERDMA_CMDQ_CQDB_REG 0x300 75 76 /* DEV_CTRL_REG details. */ 77 #define ERDMA_REG_DEV_CTRL_RESET_MASK 0x00000001 78 #define ERDMA_REG_DEV_CTRL_INIT_MASK 0x00000002 79 80 /* DEV_ST_REG details. */ 81 #define ERDMA_REG_DEV_ST_RESET_DONE_MASK 0x00000001U 82 #define ERDMA_REG_DEV_ST_INIT_DONE_MASK 0x00000002U 83 84 /* eRDMA PCIe DBs definition. */ 85 #define ERDMA_BAR_DB_SPACE_BASE 4096 86 87 #define ERDMA_BAR_SQDB_SPACE_OFFSET ERDMA_BAR_DB_SPACE_BASE 88 #define ERDMA_BAR_SQDB_SPACE_SIZE (384 * 1024) 89 90 #define ERDMA_BAR_RQDB_SPACE_OFFSET \ 91 (ERDMA_BAR_SQDB_SPACE_OFFSET + ERDMA_BAR_SQDB_SPACE_SIZE) 92 #define ERDMA_BAR_RQDB_SPACE_SIZE (96 * 1024) 93 94 #define ERDMA_BAR_CQDB_SPACE_OFFSET \ 95 (ERDMA_BAR_RQDB_SPACE_OFFSET + ERDMA_BAR_RQDB_SPACE_SIZE) 96 97 #define ERDMA_SDB_SHARED_PAGE_INDEX 95 98 99 /* Doorbell related. */ 100 #define ERDMA_DB_SIZE 8 101 102 #define ERDMA_CQDB_IDX_MASK GENMASK_ULL(63, 56) 103 #define ERDMA_CQDB_CQN_MASK GENMASK_ULL(55, 32) 104 #define ERDMA_CQDB_ARM_MASK BIT_ULL(31) 105 #define ERDMA_CQDB_SOL_MASK BIT_ULL(30) 106 #define ERDMA_CQDB_CMDSN_MASK GENMASK_ULL(29, 28) 107 #define ERDMA_CQDB_CI_MASK GENMASK_ULL(23, 0) 108 109 #define ERDMA_EQDB_ARM_MASK BIT(31) 110 #define ERDMA_EQDB_CI_MASK GENMASK_ULL(23, 0) 111 112 #define ERDMA_PAGE_SIZE_SUPPORT 0x7FFFF000 113 114 /* Hardware page size definition */ 115 #define ERDMA_HW_PAGE_SHIFT 12 116 #define ERDMA_HW_PAGE_SIZE 4096 117 118 /* WQE related. */ 119 #define EQE_SIZE 16 120 #define EQE_SHIFT 4 121 #define RQE_SIZE 32 122 #define RQE_SHIFT 5 123 #define CQE_SIZE 32 124 #define CQE_SHIFT 5 125 #define SQEBB_SIZE 32 126 #define SQEBB_SHIFT 5 127 #define SQEBB_MASK (~(SQEBB_SIZE - 1)) 128 #define SQEBB_ALIGN(size) ((size + SQEBB_SIZE - 1) & SQEBB_MASK) 129 #define SQEBB_COUNT(size) (SQEBB_ALIGN(size) >> SQEBB_SHIFT) 130 131 #define ERDMA_MAX_SQE_SIZE 128 132 #define ERDMA_MAX_WQEBB_PER_SQE 4 133 134 /* CMDQ related. */ 135 #define ERDMA_CMDQ_MAX_OUTSTANDING 128 136 #define ERDMA_CMDQ_SQE_SIZE 128 137 138 /* cmdq sub module definition. */ 139 enum CMDQ_WQE_SUB_MOD { 140 CMDQ_SUBMOD_RDMA = 0, 141 CMDQ_SUBMOD_COMMON = 1 142 }; 143 144 enum CMDQ_RDMA_OPCODE { 145 CMDQ_OPCODE_QUERY_DEVICE = 0, 146 CMDQ_OPCODE_CREATE_QP = 1, 147 CMDQ_OPCODE_DESTROY_QP = 2, 148 CMDQ_OPCODE_MODIFY_QP = 3, 149 CMDQ_OPCODE_CREATE_CQ = 4, 150 CMDQ_OPCODE_DESTROY_CQ = 5, 151 CMDQ_OPCODE_REFLUSH = 6, 152 CMDQ_OPCODE_REG_MR = 8, 153 CMDQ_OPCODE_DEREG_MR = 9, 154 CMDQ_OPCODE_SET_GID = 14, 155 CMDQ_OPCODE_CREATE_AH = 15, 156 CMDQ_OPCODE_DESTROY_AH = 16, 157 CMDQ_OPCODE_QUERY_QP = 17, 158 }; 159 160 enum CMDQ_COMMON_OPCODE { 161 CMDQ_OPCODE_CREATE_EQ = 0, 162 CMDQ_OPCODE_DESTROY_EQ = 1, 163 CMDQ_OPCODE_QUERY_FW_INFO = 2, 164 CMDQ_OPCODE_CONF_MTU = 3, 165 CMDQ_OPCODE_GET_STATS = 4, 166 CMDQ_OPCODE_CONF_DEVICE = 5, 167 CMDQ_OPCODE_ALLOC_DB = 8, 168 CMDQ_OPCODE_FREE_DB = 9, 169 }; 170 171 /* cmdq-SQE HDR */ 172 #define ERDMA_CMD_HDR_WQEBB_CNT_MASK GENMASK_ULL(54, 52) 173 #define ERDMA_CMD_HDR_CONTEXT_COOKIE_MASK GENMASK_ULL(47, 32) 174 #define ERDMA_CMD_HDR_SUB_MOD_MASK GENMASK_ULL(25, 24) 175 #define ERDMA_CMD_HDR_OPCODE_MASK GENMASK_ULL(23, 16) 176 #define ERDMA_CMD_HDR_WQEBB_INDEX_MASK GENMASK_ULL(15, 0) 177 178 struct erdma_cmdq_destroy_cq_req { 179 u64 hdr; 180 u32 cqn; 181 }; 182 183 #define ERDMA_EQ_TYPE_AEQ 0 184 #define ERDMA_EQ_TYPE_CEQ 1 185 186 struct erdma_cmdq_create_eq_req { 187 u64 hdr; 188 u64 qbuf_addr; 189 u8 vector_idx; 190 u8 eqn; 191 u8 depth; 192 u8 qtype; 193 u32 db_dma_addr_l; 194 u32 db_dma_addr_h; 195 }; 196 197 struct erdma_cmdq_destroy_eq_req { 198 u64 hdr; 199 u64 rsvd0; 200 u8 vector_idx; 201 u8 eqn; 202 u8 rsvd1; 203 u8 qtype; 204 }; 205 206 /* config device cfg */ 207 #define ERDMA_CMD_CONFIG_DEVICE_PS_EN_MASK BIT(31) 208 #define ERDMA_CMD_CONFIG_DEVICE_PGSHIFT_MASK GENMASK(4, 0) 209 210 struct erdma_cmdq_config_device_req { 211 u64 hdr; 212 u32 cfg; 213 u32 rsvd[5]; 214 }; 215 216 struct erdma_cmdq_config_mtu_req { 217 u64 hdr; 218 u32 mtu; 219 }; 220 221 /* ext db requests(alloc and free) cfg */ 222 #define ERDMA_CMD_EXT_DB_CQ_EN_MASK BIT(2) 223 #define ERDMA_CMD_EXT_DB_RQ_EN_MASK BIT(1) 224 #define ERDMA_CMD_EXT_DB_SQ_EN_MASK BIT(0) 225 226 struct erdma_cmdq_ext_db_req { 227 u64 hdr; 228 u32 cfg; 229 u16 rdb_off; 230 u16 sdb_off; 231 u16 rsvd0; 232 u16 cdb_off; 233 u32 rsvd1[3]; 234 }; 235 236 /* alloc db response qword 0 definition */ 237 #define ERDMA_CMD_ALLOC_DB_RESP_RDB_MASK GENMASK_ULL(63, 48) 238 #define ERDMA_CMD_ALLOC_DB_RESP_CDB_MASK GENMASK_ULL(47, 32) 239 #define ERDMA_CMD_ALLOC_DB_RESP_SDB_MASK GENMASK_ULL(15, 0) 240 241 /* create_cq cfg0 */ 242 #define ERDMA_CMD_CREATE_CQ_DEPTH_MASK GENMASK(31, 24) 243 #define ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK GENMASK(23, 20) 244 #define ERDMA_CMD_CREATE_CQ_CQN_MASK GENMASK(19, 0) 245 246 /* create_cq cfg1 */ 247 #define ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK GENMASK(31, 16) 248 #define ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK BIT(15) 249 #define ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK BIT(11) 250 #define ERDMA_CMD_CREATE_CQ_EQN_MASK GENMASK(9, 0) 251 252 /* create_cq cfg2 */ 253 #define ERDMA_CMD_CREATE_CQ_DB_CFG_MASK GENMASK(15, 0) 254 255 struct erdma_cmdq_create_cq_req { 256 u64 hdr; 257 u32 cfg0; 258 u32 qbuf_addr_l; 259 u32 qbuf_addr_h; 260 u32 cfg1; 261 u64 cq_dbrec_dma; 262 u32 first_page_offset; 263 u32 cfg2; 264 }; 265 266 /* regmr/deregmr cfg0 */ 267 #define ERDMA_CMD_MR_VALID_MASK BIT(31) 268 #define ERDMA_CMD_MR_VERSION_MASK GENMASK(30, 28) 269 #define ERDMA_CMD_MR_KEY_MASK GENMASK(27, 20) 270 #define ERDMA_CMD_MR_MPT_IDX_MASK GENMASK(19, 0) 271 272 /* regmr cfg1 */ 273 #define ERDMA_CMD_REGMR_PD_MASK GENMASK(31, 12) 274 #define ERDMA_CMD_REGMR_TYPE_MASK GENMASK(7, 6) 275 #define ERDMA_CMD_REGMR_RIGHT_MASK GENMASK(5, 1) 276 277 /* regmr cfg2 */ 278 #define ERDMA_CMD_REGMR_PAGESIZE_MASK GENMASK(31, 27) 279 #define ERDMA_CMD_REGMR_MTT_PAGESIZE_MASK GENMASK(26, 24) 280 #define ERDMA_CMD_REGMR_MTT_LEVEL_MASK GENMASK(21, 20) 281 #define ERDMA_CMD_REGMR_MTT_CNT_MASK GENMASK(19, 0) 282 283 struct erdma_cmdq_reg_mr_req { 284 u64 hdr; 285 u32 cfg0; 286 u32 cfg1; 287 u64 start_va; 288 u32 size; 289 u32 cfg2; 290 union { 291 u64 phy_addr[4]; 292 struct { 293 u64 rsvd; 294 u32 size_h; 295 u32 mtt_cnt_h; 296 }; 297 }; 298 }; 299 300 struct erdma_cmdq_dereg_mr_req { 301 u64 hdr; 302 u32 cfg; 303 }; 304 305 /* create_av cfg0 */ 306 #define ERDMA_CMD_CREATE_AV_FL_MASK GENMASK(19, 0) 307 #define ERDMA_CMD_CREATE_AV_NTYPE_MASK BIT(20) 308 309 struct erdma_av_cfg { 310 u32 cfg0; 311 u8 traffic_class; 312 u8 hop_limit; 313 u8 sl; 314 u8 rsvd; 315 u16 udp_sport; 316 u16 sgid_index; 317 u8 dmac[ETH_ALEN]; 318 u8 padding[2]; 319 u8 dgid[ERDMA_ROCEV2_GID_SIZE]; 320 }; 321 322 struct erdma_cmdq_create_ah_req { 323 u64 hdr; 324 u32 pdn; 325 u32 ahn; 326 struct erdma_av_cfg av_cfg; 327 }; 328 329 struct erdma_cmdq_destroy_ah_req { 330 u64 hdr; 331 u32 pdn; 332 u32 ahn; 333 }; 334 335 /* modify qp cfg */ 336 #define ERDMA_CMD_MODIFY_QP_STATE_MASK GENMASK(31, 24) 337 #define ERDMA_CMD_MODIFY_QP_CC_MASK GENMASK(23, 20) 338 #define ERDMA_CMD_MODIFY_QP_QPN_MASK GENMASK(19, 0) 339 340 struct erdma_cmdq_modify_qp_req { 341 u64 hdr; 342 u32 cfg; 343 u32 cookie; 344 __be32 dip; 345 __be32 sip; 346 __be16 sport; 347 __be16 dport; 348 u32 send_nxt; 349 u32 recv_nxt; 350 }; 351 352 /* modify qp cfg1 for roce device */ 353 #define ERDMA_CMD_MODIFY_QP_DQPN_MASK GENMASK(19, 0) 354 355 struct erdma_cmdq_mod_qp_req_rocev2 { 356 u64 hdr; 357 u32 cfg0; 358 u32 cfg1; 359 u32 attr_mask; 360 u32 qkey; 361 u32 rq_psn; 362 u32 sq_psn; 363 struct erdma_av_cfg av_cfg; 364 }; 365 366 /* query qp response mask */ 367 #define ERDMA_CMD_QUERY_QP_RESP_SQ_PSN_MASK GENMASK_ULL(23, 0) 368 #define ERDMA_CMD_QUERY_QP_RESP_RQ_PSN_MASK GENMASK_ULL(47, 24) 369 #define ERDMA_CMD_QUERY_QP_RESP_QP_STATE_MASK GENMASK_ULL(55, 48) 370 #define ERDMA_CMD_QUERY_QP_RESP_SQ_DRAINING_MASK GENMASK_ULL(56, 56) 371 372 struct erdma_cmdq_query_qp_req_rocev2 { 373 u64 hdr; 374 u32 qpn; 375 }; 376 377 enum erdma_qp_type { 378 ERDMA_QPT_RC = 0, 379 ERDMA_QPT_UD = 1, 380 }; 381 382 /* create qp cfg0 */ 383 #define ERDMA_CMD_CREATE_QP_SQ_DEPTH_MASK GENMASK(31, 20) 384 #define ERDMA_CMD_CREATE_QP_QPN_MASK GENMASK(19, 0) 385 386 /* create qp cfg1 */ 387 #define ERDMA_CMD_CREATE_QP_RQ_DEPTH_MASK GENMASK(31, 20) 388 #define ERDMA_CMD_CREATE_QP_PD_MASK GENMASK(19, 0) 389 390 /* create qp cfg2 */ 391 #define ERDMA_CMD_CREATE_QP_TYPE_MASK GENMASK(3, 0) 392 393 /* create qp cqn_mtt_cfg */ 394 #define ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK GENMASK(31, 28) 395 #define ERDMA_CMD_CREATE_QP_DB_CFG_MASK BIT(25) 396 #define ERDMA_CMD_CREATE_QP_CQN_MASK GENMASK(23, 0) 397 398 /* create qp mtt_cfg */ 399 #define ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK GENMASK(31, 12) 400 #define ERDMA_CMD_CREATE_QP_MTT_CNT_MASK GENMASK(11, 1) 401 #define ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK BIT(0) 402 403 /* create qp db cfg */ 404 #define ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK GENMASK(31, 16) 405 #define ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK GENMASK(15, 0) 406 407 #define ERDMA_CMDQ_CREATE_QP_RESP_COOKIE_MASK GENMASK_ULL(31, 0) 408 409 struct erdma_cmdq_create_qp_req { 410 u64 hdr; 411 u32 cfg0; 412 u32 cfg1; 413 u32 sq_cqn_mtt_cfg; 414 u32 rq_cqn_mtt_cfg; 415 u64 sq_buf_addr; 416 u64 rq_buf_addr; 417 u32 sq_mtt_cfg; 418 u32 rq_mtt_cfg; 419 u64 sq_dbrec_dma; 420 u64 rq_dbrec_dma; 421 422 u64 sq_mtt_entry[3]; 423 u64 rq_mtt_entry[3]; 424 425 u32 db_cfg; 426 u32 cfg2; 427 }; 428 429 struct erdma_cmdq_destroy_qp_req { 430 u64 hdr; 431 u32 qpn; 432 }; 433 434 struct erdma_cmdq_reflush_req { 435 u64 hdr; 436 u32 qpn; 437 u32 sq_pi; 438 u32 rq_pi; 439 }; 440 441 #define ERDMA_HW_RESP_SIZE 256 442 443 struct erdma_cmdq_query_req { 444 u64 hdr; 445 u32 rsvd; 446 u32 index; 447 448 u64 target_addr; 449 u32 target_length; 450 }; 451 452 #define ERDMA_HW_RESP_MAGIC 0x5566 453 454 struct erdma_cmdq_query_resp_hdr { 455 u16 magic; 456 u8 ver; 457 u8 length; 458 459 u32 index; 460 u32 rsvd[2]; 461 }; 462 463 struct erdma_cmdq_query_stats_resp { 464 struct erdma_cmdq_query_resp_hdr hdr; 465 466 u64 tx_req_cnt; 467 u64 tx_packets_cnt; 468 u64 tx_bytes_cnt; 469 u64 tx_drop_packets_cnt; 470 u64 tx_bps_meter_drop_packets_cnt; 471 u64 tx_pps_meter_drop_packets_cnt; 472 u64 rx_packets_cnt; 473 u64 rx_bytes_cnt; 474 u64 rx_drop_packets_cnt; 475 u64 rx_bps_meter_drop_packets_cnt; 476 u64 rx_pps_meter_drop_packets_cnt; 477 }; 478 479 enum erdma_network_type { 480 ERDMA_NETWORK_TYPE_IPV4 = 0, 481 ERDMA_NETWORK_TYPE_IPV6 = 1, 482 }; 483 484 enum erdma_set_gid_op { 485 ERDMA_SET_GID_OP_ADD = 0, 486 ERDMA_SET_GID_OP_DEL = 1, 487 }; 488 489 /* set gid cfg */ 490 #define ERDMA_CMD_SET_GID_SGID_IDX_MASK GENMASK(15, 0) 491 #define ERDMA_CMD_SET_GID_NTYPE_MASK BIT(16) 492 #define ERDMA_CMD_SET_GID_OP_MASK BIT(31) 493 494 struct erdma_cmdq_set_gid_req { 495 u64 hdr; 496 u32 cfg; 497 u8 gid[ERDMA_ROCEV2_GID_SIZE]; 498 }; 499 500 /* cap qword 0 definition */ 501 #define ERDMA_CMD_DEV_CAP_MAX_GID_MASK GENMASK_ULL(51, 48) 502 #define ERDMA_CMD_DEV_CAP_MAX_CQE_MASK GENMASK_ULL(47, 40) 503 #define ERDMA_CMD_DEV_CAP_FLAGS_MASK GENMASK_ULL(31, 24) 504 #define ERDMA_CMD_DEV_CAP_MAX_RECV_WR_MASK GENMASK_ULL(23, 16) 505 #define ERDMA_CMD_DEV_CAP_MAX_AH_MASK GENMASK_ULL(15, 8) 506 #define ERDMA_CMD_DEV_CAP_MAX_MR_SIZE_MASK GENMASK_ULL(7, 0) 507 508 /* cap qword 1 definition */ 509 #define ERDMA_CMD_DEV_CAP_DMA_LOCAL_KEY_MASK GENMASK_ULL(63, 32) 510 #define ERDMA_CMD_DEV_CAP_DEFAULT_CC_MASK GENMASK_ULL(31, 28) 511 #define ERDMA_CMD_DEV_CAP_QBLOCK_MASK GENMASK_ULL(27, 16) 512 #define ERDMA_CMD_DEV_CAP_MAX_MW_MASK GENMASK_ULL(7, 0) 513 514 #define ERDMA_NQP_PER_QBLOCK 1024 515 516 enum { 517 ERDMA_DEV_CAP_FLAGS_ATOMIC = 1 << 7, 518 ERDMA_DEV_CAP_FLAGS_MTT_VA = 1 << 5, 519 ERDMA_DEV_CAP_FLAGS_EXTEND_DB = 1 << 3, 520 }; 521 522 #define ERDMA_CMD_INFO0_FW_VER_MASK GENMASK_ULL(31, 0) 523 524 /* CQE hdr */ 525 #define ERDMA_CQE_HDR_OWNER_MASK BIT(31) 526 #define ERDMA_CQE_HDR_OPCODE_MASK GENMASK(23, 16) 527 #define ERDMA_CQE_HDR_QTYPE_MASK GENMASK(15, 8) 528 #define ERDMA_CQE_HDR_SYNDROME_MASK GENMASK(7, 0) 529 530 #define ERDMA_CQE_QTYPE_SQ 0 531 #define ERDMA_CQE_QTYPE_RQ 1 532 #define ERDMA_CQE_QTYPE_CMDQ 2 533 534 #define ERDMA_CQE_NTYPE_MASK BIT(31) 535 #define ERDMA_CQE_SL_MASK GENMASK(27, 20) 536 #define ERDMA_CQE_SQPN_MASK GENMASK(19, 0) 537 538 struct erdma_cqe { 539 __be32 hdr; 540 __be32 qe_idx; 541 __be32 qpn; 542 union { 543 __le32 imm_data; 544 __be32 inv_rkey; 545 }; 546 __be32 size; 547 union { 548 struct { 549 __be32 rsvd[3]; 550 } rc; 551 552 struct { 553 __be32 rsvd[2]; 554 __be32 info; 555 } ud; 556 }; 557 }; 558 559 struct erdma_sge { 560 __aligned_le64 addr; 561 __le32 length; 562 __le32 key; 563 }; 564 565 /* Receive Queue Element */ 566 struct erdma_rqe { 567 __le16 qe_idx; 568 __le16 rsvd0; 569 __le32 qpn; 570 __le32 rsvd1; 571 __le32 rsvd2; 572 __le64 to; 573 __le32 length; 574 __le32 stag; 575 }; 576 577 /* SQE */ 578 #define ERDMA_SQE_HDR_SGL_LEN_MASK GENMASK_ULL(63, 56) 579 #define ERDMA_SQE_HDR_WQEBB_CNT_MASK GENMASK_ULL(54, 52) 580 #define ERDMA_SQE_HDR_QPN_MASK GENMASK_ULL(51, 32) 581 #define ERDMA_SQE_HDR_OPCODE_MASK GENMASK_ULL(31, 27) 582 #define ERDMA_SQE_HDR_DWQE_MASK BIT_ULL(26) 583 #define ERDMA_SQE_HDR_INLINE_MASK BIT_ULL(25) 584 #define ERDMA_SQE_HDR_FENCE_MASK BIT_ULL(24) 585 #define ERDMA_SQE_HDR_SE_MASK BIT_ULL(23) 586 #define ERDMA_SQE_HDR_CE_MASK BIT_ULL(22) 587 #define ERDMA_SQE_HDR_WQEBB_INDEX_MASK GENMASK_ULL(15, 0) 588 589 /* REG MR attrs */ 590 #define ERDMA_SQE_MR_ACCESS_MASK GENMASK(5, 1) 591 #define ERDMA_SQE_MR_MTT_TYPE_MASK GENMASK(7, 6) 592 #define ERDMA_SQE_MR_MTT_CNT_MASK GENMASK(31, 12) 593 594 struct erdma_write_sqe { 595 __le64 hdr; 596 __be32 imm_data; 597 __le32 length; 598 599 __le32 sink_stag; 600 __le32 sink_to_l; 601 __le32 sink_to_h; 602 603 __le32 rsvd; 604 605 struct erdma_sge sgl[]; 606 }; 607 608 struct erdma_send_sqe_rc { 609 __le64 hdr; 610 union { 611 __be32 imm_data; 612 __le32 invalid_stag; 613 }; 614 615 __le32 length; 616 struct erdma_sge sgl[]; 617 }; 618 619 struct erdma_send_sqe_ud { 620 __le64 hdr; 621 __be32 imm_data; 622 __le32 length; 623 __le32 qkey; 624 __le32 dst_qpn; 625 __le32 ahn; 626 __le32 rsvd; 627 struct erdma_sge sgl[]; 628 }; 629 630 struct erdma_readreq_sqe { 631 __le64 hdr; 632 __le32 invalid_stag; 633 __le32 length; 634 __le32 sink_stag; 635 __le32 sink_to_l; 636 __le32 sink_to_h; 637 __le32 rsvd; 638 }; 639 640 struct erdma_atomic_sqe { 641 __le64 hdr; 642 __le64 rsvd; 643 __le64 fetchadd_swap_data; 644 __le64 cmp_data; 645 646 struct erdma_sge remote; 647 struct erdma_sge sgl; 648 }; 649 650 struct erdma_reg_mr_sqe { 651 __le64 hdr; 652 __le64 addr; 653 __le32 length; 654 __le32 stag; 655 __le32 attrs; 656 __le32 rsvd; 657 }; 658 659 /* EQ related. */ 660 #define ERDMA_DEFAULT_EQ_DEPTH 4096 661 662 /* ceqe */ 663 #define ERDMA_CEQE_HDR_DB_MASK BIT_ULL(63) 664 #define ERDMA_CEQE_HDR_PI_MASK GENMASK_ULL(55, 32) 665 #define ERDMA_CEQE_HDR_O_MASK BIT_ULL(31) 666 #define ERDMA_CEQE_HDR_CQN_MASK GENMASK_ULL(19, 0) 667 668 /* aeqe */ 669 #define ERDMA_AEQE_HDR_O_MASK BIT(31) 670 #define ERDMA_AEQE_HDR_TYPE_MASK GENMASK(23, 16) 671 #define ERDMA_AEQE_HDR_SUBTYPE_MASK GENMASK(7, 0) 672 673 #define ERDMA_AE_TYPE_QP_FATAL_EVENT 0 674 #define ERDMA_AE_TYPE_QP_ERQ_ERR_EVENT 1 675 #define ERDMA_AE_TYPE_ACC_ERR_EVENT 2 676 #define ERDMA_AE_TYPE_CQ_ERR 3 677 #define ERDMA_AE_TYPE_OTHER_ERROR 4 678 679 struct erdma_aeqe { 680 __le32 hdr; 681 __le32 event_data0; 682 __le32 event_data1; 683 __le32 rsvd; 684 }; 685 686 enum erdma_opcode { 687 ERDMA_OP_WRITE = 0, 688 ERDMA_OP_READ = 1, 689 ERDMA_OP_SEND = 2, 690 ERDMA_OP_SEND_WITH_IMM = 3, 691 692 ERDMA_OP_RECEIVE = 4, 693 ERDMA_OP_RECV_IMM = 5, 694 ERDMA_OP_RECV_INV = 6, 695 696 ERDMA_OP_RSVD0 = 7, 697 ERDMA_OP_RSVD1 = 8, 698 ERDMA_OP_WRITE_WITH_IMM = 9, 699 700 ERDMA_OP_RSVD2 = 10, 701 ERDMA_OP_RSVD3 = 11, 702 703 ERDMA_OP_RSP_SEND_IMM = 12, 704 ERDMA_OP_SEND_WITH_INV = 13, 705 706 ERDMA_OP_REG_MR = 14, 707 ERDMA_OP_LOCAL_INV = 15, 708 ERDMA_OP_READ_WITH_INV = 16, 709 ERDMA_OP_ATOMIC_CAS = 17, 710 ERDMA_OP_ATOMIC_FAA = 18, 711 ERDMA_NUM_OPCODES = 19, 712 ERDMA_OP_INVALID = ERDMA_NUM_OPCODES + 1 713 }; 714 715 enum erdma_wc_status { 716 ERDMA_WC_SUCCESS = 0, 717 ERDMA_WC_GENERAL_ERR = 1, 718 ERDMA_WC_RECV_WQE_FORMAT_ERR = 2, 719 ERDMA_WC_RECV_STAG_INVALID_ERR = 3, 720 ERDMA_WC_RECV_ADDR_VIOLATION_ERR = 4, 721 ERDMA_WC_RECV_RIGHT_VIOLATION_ERR = 5, 722 ERDMA_WC_RECV_PDID_ERR = 6, 723 ERDMA_WC_RECV_WARRPING_ERR = 7, 724 ERDMA_WC_SEND_WQE_FORMAT_ERR = 8, 725 ERDMA_WC_SEND_WQE_ORD_EXCEED = 9, 726 ERDMA_WC_SEND_STAG_INVALID_ERR = 10, 727 ERDMA_WC_SEND_ADDR_VIOLATION_ERR = 11, 728 ERDMA_WC_SEND_RIGHT_VIOLATION_ERR = 12, 729 ERDMA_WC_SEND_PDID_ERR = 13, 730 ERDMA_WC_SEND_WARRPING_ERR = 14, 731 ERDMA_WC_FLUSH_ERR = 15, 732 ERDMA_WC_RETRY_EXC_ERR = 16, 733 ERDMA_NUM_WC_STATUS 734 }; 735 736 enum erdma_vendor_err { 737 ERDMA_WC_VENDOR_NO_ERR = 0, 738 ERDMA_WC_VENDOR_INVALID_RQE = 1, 739 ERDMA_WC_VENDOR_RQE_INVALID_STAG = 2, 740 ERDMA_WC_VENDOR_RQE_ADDR_VIOLATION = 3, 741 ERDMA_WC_VENDOR_RQE_ACCESS_RIGHT_ERR = 4, 742 ERDMA_WC_VENDOR_RQE_INVALID_PD = 5, 743 ERDMA_WC_VENDOR_RQE_WRAP_ERR = 6, 744 ERDMA_WC_VENDOR_INVALID_SQE = 0x20, 745 ERDMA_WC_VENDOR_ZERO_ORD = 0x21, 746 ERDMA_WC_VENDOR_SQE_INVALID_STAG = 0x30, 747 ERDMA_WC_VENDOR_SQE_ADDR_VIOLATION = 0x31, 748 ERDMA_WC_VENDOR_SQE_ACCESS_ERR = 0x32, 749 ERDMA_WC_VENDOR_SQE_INVALID_PD = 0x33, 750 ERDMA_WC_VENDOR_SQE_WARP_ERR = 0x34 751 }; 752 753 #endif 754