1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2 
3 /* Authors: Cheng Xu <[email protected]> */
4 /*          Kai Shen <[email protected]> */
5 /* Copyright (c) 2020-2022, Alibaba Group. */
6 
7 #ifndef __ERDMA_HW_H__
8 #define __ERDMA_HW_H__
9 
10 #include <linux/kernel.h>
11 #include <linux/types.h>
12 #include <linux/if_ether.h>
13 
14 /* PCIe device related definition. */
15 #define ERDMA_PCI_WIDTH 64
16 #define ERDMA_FUNC_BAR 0
17 #define ERDMA_MISX_BAR 2
18 
19 #define ERDMA_BAR_MASK (BIT(ERDMA_FUNC_BAR) | BIT(ERDMA_MISX_BAR))
20 
21 /* MSI-X related. */
22 #define ERDMA_NUM_MSIX_VEC 32U
23 #define ERDMA_MSIX_VECTOR_CMDQ 0
24 
25 /* RoCEv2 related */
26 #define ERDMA_ROCEV2_GID_SIZE 16
27 #define ERDMA_MAX_PKEYS 1
28 #define ERDMA_DEFAULT_PKEY 0xFFFF
29 
30 /* erdma device protocol type */
31 enum erdma_proto_type {
32 	ERDMA_PROTO_IWARP = 0,
33 	ERDMA_PROTO_ROCEV2 = 1,
34 	ERDMA_PROTO_COUNT = 2,
35 };
36 
37 /* PCIe Bar0 Registers. */
38 #define ERDMA_REGS_VERSION_REG 0x0
39 #define ERDMA_REGS_DEV_PROTO_REG 0xC
40 #define ERDMA_REGS_DEV_CTRL_REG 0x10
41 #define ERDMA_REGS_DEV_ST_REG 0x14
42 #define ERDMA_REGS_NETDEV_MAC_L_REG 0x18
43 #define ERDMA_REGS_NETDEV_MAC_H_REG 0x1C
44 #define ERDMA_REGS_CMDQ_SQ_ADDR_L_REG 0x20
45 #define ERDMA_REGS_CMDQ_SQ_ADDR_H_REG 0x24
46 #define ERDMA_REGS_CMDQ_CQ_ADDR_L_REG 0x28
47 #define ERDMA_REGS_CMDQ_CQ_ADDR_H_REG 0x2C
48 #define ERDMA_REGS_CMDQ_DEPTH_REG 0x30
49 #define ERDMA_REGS_CMDQ_EQ_DEPTH_REG 0x34
50 #define ERDMA_REGS_CMDQ_EQ_ADDR_L_REG 0x38
51 #define ERDMA_REGS_CMDQ_EQ_ADDR_H_REG 0x3C
52 #define ERDMA_REGS_AEQ_ADDR_L_REG 0x40
53 #define ERDMA_REGS_AEQ_ADDR_H_REG 0x44
54 #define ERDMA_REGS_AEQ_DEPTH_REG 0x48
55 #define ERDMA_REGS_GRP_NUM_REG 0x4c
56 #define ERDMA_REGS_AEQ_DB_REG 0x50
57 #define ERDMA_CMDQ_SQ_DB_HOST_ADDR_REG 0x60
58 #define ERDMA_CMDQ_CQ_DB_HOST_ADDR_REG 0x68
59 #define ERDMA_CMDQ_EQ_DB_HOST_ADDR_REG 0x70
60 #define ERDMA_AEQ_DB_HOST_ADDR_REG 0x78
61 #define ERDMA_REGS_STATS_TSO_IN_PKTS_REG 0x80
62 #define ERDMA_REGS_STATS_TSO_OUT_PKTS_REG 0x88
63 #define ERDMA_REGS_STATS_TSO_OUT_BYTES_REG 0x90
64 #define ERDMA_REGS_STATS_TX_DROP_PKTS_REG 0x98
65 #define ERDMA_REGS_STATS_TX_BPS_METER_DROP_PKTS_REG 0xa0
66 #define ERDMA_REGS_STATS_TX_PPS_METER_DROP_PKTS_REG 0xa8
67 #define ERDMA_REGS_STATS_RX_PKTS_REG 0xc0
68 #define ERDMA_REGS_STATS_RX_BYTES_REG 0xc8
69 #define ERDMA_REGS_STATS_RX_DROP_PKTS_REG 0xd0
70 #define ERDMA_REGS_STATS_RX_BPS_METER_DROP_PKTS_REG 0xd8
71 #define ERDMA_REGS_STATS_RX_PPS_METER_DROP_PKTS_REG 0xe0
72 #define ERDMA_REGS_CEQ_DB_BASE_REG 0x100
73 #define ERDMA_CMDQ_SQDB_REG 0x200
74 #define ERDMA_CMDQ_CQDB_REG 0x300
75 
76 /* DEV_CTRL_REG details. */
77 #define ERDMA_REG_DEV_CTRL_RESET_MASK 0x00000001
78 #define ERDMA_REG_DEV_CTRL_INIT_MASK 0x00000002
79 
80 /* DEV_ST_REG details. */
81 #define ERDMA_REG_DEV_ST_RESET_DONE_MASK 0x00000001U
82 #define ERDMA_REG_DEV_ST_INIT_DONE_MASK 0x00000002U
83 
84 /* eRDMA PCIe DBs definition. */
85 #define ERDMA_BAR_DB_SPACE_BASE 4096
86 
87 #define ERDMA_BAR_SQDB_SPACE_OFFSET ERDMA_BAR_DB_SPACE_BASE
88 #define ERDMA_BAR_SQDB_SPACE_SIZE (384 * 1024)
89 
90 #define ERDMA_BAR_RQDB_SPACE_OFFSET \
91 	(ERDMA_BAR_SQDB_SPACE_OFFSET + ERDMA_BAR_SQDB_SPACE_SIZE)
92 #define ERDMA_BAR_RQDB_SPACE_SIZE (96 * 1024)
93 
94 #define ERDMA_BAR_CQDB_SPACE_OFFSET \
95 	(ERDMA_BAR_RQDB_SPACE_OFFSET + ERDMA_BAR_RQDB_SPACE_SIZE)
96 
97 #define ERDMA_SDB_SHARED_PAGE_INDEX 95
98 
99 /* Doorbell related. */
100 #define ERDMA_DB_SIZE 8
101 
102 #define ERDMA_CQDB_IDX_MASK GENMASK_ULL(63, 56)
103 #define ERDMA_CQDB_CQN_MASK GENMASK_ULL(55, 32)
104 #define ERDMA_CQDB_ARM_MASK BIT_ULL(31)
105 #define ERDMA_CQDB_SOL_MASK BIT_ULL(30)
106 #define ERDMA_CQDB_CMDSN_MASK GENMASK_ULL(29, 28)
107 #define ERDMA_CQDB_CI_MASK GENMASK_ULL(23, 0)
108 
109 #define ERDMA_EQDB_ARM_MASK BIT(31)
110 #define ERDMA_EQDB_CI_MASK GENMASK_ULL(23, 0)
111 
112 #define ERDMA_PAGE_SIZE_SUPPORT 0x7FFFF000
113 
114 /* Hardware page size definition */
115 #define ERDMA_HW_PAGE_SHIFT 12
116 #define ERDMA_HW_PAGE_SIZE 4096
117 
118 /* WQE related. */
119 #define EQE_SIZE 16
120 #define EQE_SHIFT 4
121 #define RQE_SIZE 32
122 #define RQE_SHIFT 5
123 #define CQE_SIZE 32
124 #define CQE_SHIFT 5
125 #define SQEBB_SIZE 32
126 #define SQEBB_SHIFT 5
127 #define SQEBB_MASK (~(SQEBB_SIZE - 1))
128 #define SQEBB_ALIGN(size) ((size + SQEBB_SIZE - 1) & SQEBB_MASK)
129 #define SQEBB_COUNT(size) (SQEBB_ALIGN(size) >> SQEBB_SHIFT)
130 
131 #define ERDMA_MAX_SQE_SIZE 128
132 #define ERDMA_MAX_WQEBB_PER_SQE 4
133 
134 /* CMDQ related. */
135 #define ERDMA_CMDQ_MAX_OUTSTANDING 128
136 #define ERDMA_CMDQ_SQE_SIZE 128
137 
138 /* cmdq sub module definition. */
139 enum CMDQ_WQE_SUB_MOD {
140 	CMDQ_SUBMOD_RDMA = 0,
141 	CMDQ_SUBMOD_COMMON = 1
142 };
143 
144 enum CMDQ_RDMA_OPCODE {
145 	CMDQ_OPCODE_QUERY_DEVICE = 0,
146 	CMDQ_OPCODE_CREATE_QP = 1,
147 	CMDQ_OPCODE_DESTROY_QP = 2,
148 	CMDQ_OPCODE_MODIFY_QP = 3,
149 	CMDQ_OPCODE_CREATE_CQ = 4,
150 	CMDQ_OPCODE_DESTROY_CQ = 5,
151 	CMDQ_OPCODE_REFLUSH = 6,
152 	CMDQ_OPCODE_REG_MR = 8,
153 	CMDQ_OPCODE_DEREG_MR = 9,
154 	CMDQ_OPCODE_SET_GID = 14,
155 	CMDQ_OPCODE_CREATE_AH = 15,
156 	CMDQ_OPCODE_DESTROY_AH = 16,
157 	CMDQ_OPCODE_QUERY_QP = 17,
158 };
159 
160 enum CMDQ_COMMON_OPCODE {
161 	CMDQ_OPCODE_CREATE_EQ = 0,
162 	CMDQ_OPCODE_DESTROY_EQ = 1,
163 	CMDQ_OPCODE_QUERY_FW_INFO = 2,
164 	CMDQ_OPCODE_CONF_MTU = 3,
165 	CMDQ_OPCODE_GET_STATS = 4,
166 	CMDQ_OPCODE_CONF_DEVICE = 5,
167 	CMDQ_OPCODE_ALLOC_DB = 8,
168 	CMDQ_OPCODE_FREE_DB = 9,
169 };
170 
171 /* cmdq-SQE HDR */
172 #define ERDMA_CMD_HDR_WQEBB_CNT_MASK GENMASK_ULL(54, 52)
173 #define ERDMA_CMD_HDR_CONTEXT_COOKIE_MASK GENMASK_ULL(47, 32)
174 #define ERDMA_CMD_HDR_SUB_MOD_MASK GENMASK_ULL(25, 24)
175 #define ERDMA_CMD_HDR_OPCODE_MASK GENMASK_ULL(23, 16)
176 #define ERDMA_CMD_HDR_WQEBB_INDEX_MASK GENMASK_ULL(15, 0)
177 
178 struct erdma_cmdq_destroy_cq_req {
179 	u64 hdr;
180 	u32 cqn;
181 };
182 
183 #define ERDMA_EQ_TYPE_AEQ 0
184 #define ERDMA_EQ_TYPE_CEQ 1
185 
186 struct erdma_cmdq_create_eq_req {
187 	u64 hdr;
188 	u64 qbuf_addr;
189 	u8 vector_idx;
190 	u8 eqn;
191 	u8 depth;
192 	u8 qtype;
193 	u32 db_dma_addr_l;
194 	u32 db_dma_addr_h;
195 };
196 
197 struct erdma_cmdq_destroy_eq_req {
198 	u64 hdr;
199 	u64 rsvd0;
200 	u8 vector_idx;
201 	u8 eqn;
202 	u8 rsvd1;
203 	u8 qtype;
204 };
205 
206 /* config device cfg */
207 #define ERDMA_CMD_CONFIG_DEVICE_PS_EN_MASK BIT(31)
208 #define ERDMA_CMD_CONFIG_DEVICE_PGSHIFT_MASK GENMASK(4, 0)
209 
210 struct erdma_cmdq_config_device_req {
211 	u64 hdr;
212 	u32 cfg;
213 	u32 rsvd[5];
214 };
215 
216 struct erdma_cmdq_config_mtu_req {
217 	u64 hdr;
218 	u32 mtu;
219 };
220 
221 /* ext db requests(alloc and free) cfg */
222 #define ERDMA_CMD_EXT_DB_CQ_EN_MASK BIT(2)
223 #define ERDMA_CMD_EXT_DB_RQ_EN_MASK BIT(1)
224 #define ERDMA_CMD_EXT_DB_SQ_EN_MASK BIT(0)
225 
226 struct erdma_cmdq_ext_db_req {
227 	u64 hdr;
228 	u32 cfg;
229 	u16 rdb_off;
230 	u16 sdb_off;
231 	u16 rsvd0;
232 	u16 cdb_off;
233 	u32 rsvd1[3];
234 };
235 
236 /* alloc db response qword 0 definition */
237 #define ERDMA_CMD_ALLOC_DB_RESP_RDB_MASK GENMASK_ULL(63, 48)
238 #define ERDMA_CMD_ALLOC_DB_RESP_CDB_MASK GENMASK_ULL(47, 32)
239 #define ERDMA_CMD_ALLOC_DB_RESP_SDB_MASK GENMASK_ULL(15, 0)
240 
241 /* create_cq cfg0 */
242 #define ERDMA_CMD_CREATE_CQ_DEPTH_MASK GENMASK(31, 24)
243 #define ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK GENMASK(23, 20)
244 #define ERDMA_CMD_CREATE_CQ_CQN_MASK GENMASK(19, 0)
245 
246 /* create_cq cfg1 */
247 #define ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK GENMASK(31, 16)
248 #define ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK BIT(15)
249 #define ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK BIT(11)
250 #define ERDMA_CMD_CREATE_CQ_EQN_MASK GENMASK(9, 0)
251 
252 /* create_cq cfg2 */
253 #define ERDMA_CMD_CREATE_CQ_DB_CFG_MASK GENMASK(15, 0)
254 
255 struct erdma_cmdq_create_cq_req {
256 	u64 hdr;
257 	u32 cfg0;
258 	u32 qbuf_addr_l;
259 	u32 qbuf_addr_h;
260 	u32 cfg1;
261 	u64 cq_dbrec_dma;
262 	u32 first_page_offset;
263 	u32 cfg2;
264 };
265 
266 /* regmr/deregmr cfg0 */
267 #define ERDMA_CMD_MR_VALID_MASK BIT(31)
268 #define ERDMA_CMD_MR_VERSION_MASK GENMASK(30, 28)
269 #define ERDMA_CMD_MR_KEY_MASK GENMASK(27, 20)
270 #define ERDMA_CMD_MR_MPT_IDX_MASK GENMASK(19, 0)
271 
272 /* regmr cfg1 */
273 #define ERDMA_CMD_REGMR_PD_MASK GENMASK(31, 12)
274 #define ERDMA_CMD_REGMR_TYPE_MASK GENMASK(7, 6)
275 #define ERDMA_CMD_REGMR_RIGHT_MASK GENMASK(5, 1)
276 
277 /* regmr cfg2 */
278 #define ERDMA_CMD_REGMR_PAGESIZE_MASK GENMASK(31, 27)
279 #define ERDMA_CMD_REGMR_MTT_PAGESIZE_MASK GENMASK(26, 24)
280 #define ERDMA_CMD_REGMR_MTT_LEVEL_MASK GENMASK(21, 20)
281 #define ERDMA_CMD_REGMR_MTT_CNT_MASK GENMASK(19, 0)
282 
283 struct erdma_cmdq_reg_mr_req {
284 	u64 hdr;
285 	u32 cfg0;
286 	u32 cfg1;
287 	u64 start_va;
288 	u32 size;
289 	u32 cfg2;
290 	union {
291 		u64 phy_addr[4];
292 		struct {
293 			u64 rsvd;
294 			u32 size_h;
295 			u32 mtt_cnt_h;
296 		};
297 	};
298 };
299 
300 struct erdma_cmdq_dereg_mr_req {
301 	u64 hdr;
302 	u32 cfg;
303 };
304 
305 /* create_av cfg0 */
306 #define ERDMA_CMD_CREATE_AV_FL_MASK GENMASK(19, 0)
307 #define ERDMA_CMD_CREATE_AV_NTYPE_MASK BIT(20)
308 
309 struct erdma_av_cfg {
310 	u32 cfg0;
311 	u8 traffic_class;
312 	u8 hop_limit;
313 	u8 sl;
314 	u8 rsvd;
315 	u16 udp_sport;
316 	u16 sgid_index;
317 	u8 dmac[ETH_ALEN];
318 	u8 padding[2];
319 	u8 dgid[ERDMA_ROCEV2_GID_SIZE];
320 };
321 
322 struct erdma_cmdq_create_ah_req {
323 	u64 hdr;
324 	u32 pdn;
325 	u32 ahn;
326 	struct erdma_av_cfg av_cfg;
327 };
328 
329 struct erdma_cmdq_destroy_ah_req {
330 	u64 hdr;
331 	u32 pdn;
332 	u32 ahn;
333 };
334 
335 /* modify qp cfg */
336 #define ERDMA_CMD_MODIFY_QP_STATE_MASK GENMASK(31, 24)
337 #define ERDMA_CMD_MODIFY_QP_CC_MASK GENMASK(23, 20)
338 #define ERDMA_CMD_MODIFY_QP_QPN_MASK GENMASK(19, 0)
339 
340 struct erdma_cmdq_modify_qp_req {
341 	u64 hdr;
342 	u32 cfg;
343 	u32 cookie;
344 	__be32 dip;
345 	__be32 sip;
346 	__be16 sport;
347 	__be16 dport;
348 	u32 send_nxt;
349 	u32 recv_nxt;
350 };
351 
352 /* modify qp cfg1 for roce device */
353 #define ERDMA_CMD_MODIFY_QP_DQPN_MASK GENMASK(19, 0)
354 
355 struct erdma_cmdq_mod_qp_req_rocev2 {
356 	u64 hdr;
357 	u32 cfg0;
358 	u32 cfg1;
359 	u32 attr_mask;
360 	u32 qkey;
361 	u32 rq_psn;
362 	u32 sq_psn;
363 	struct erdma_av_cfg av_cfg;
364 };
365 
366 /* query qp response mask */
367 #define ERDMA_CMD_QUERY_QP_RESP_SQ_PSN_MASK GENMASK_ULL(23, 0)
368 #define ERDMA_CMD_QUERY_QP_RESP_RQ_PSN_MASK GENMASK_ULL(47, 24)
369 #define ERDMA_CMD_QUERY_QP_RESP_QP_STATE_MASK GENMASK_ULL(55, 48)
370 #define ERDMA_CMD_QUERY_QP_RESP_SQ_DRAINING_MASK GENMASK_ULL(56, 56)
371 
372 struct erdma_cmdq_query_qp_req_rocev2 {
373 	u64 hdr;
374 	u32 qpn;
375 };
376 
377 enum erdma_qp_type {
378 	ERDMA_QPT_RC = 0,
379 	ERDMA_QPT_UD = 1,
380 };
381 
382 /* create qp cfg0 */
383 #define ERDMA_CMD_CREATE_QP_SQ_DEPTH_MASK GENMASK(31, 20)
384 #define ERDMA_CMD_CREATE_QP_QPN_MASK GENMASK(19, 0)
385 
386 /* create qp cfg1 */
387 #define ERDMA_CMD_CREATE_QP_RQ_DEPTH_MASK GENMASK(31, 20)
388 #define ERDMA_CMD_CREATE_QP_PD_MASK GENMASK(19, 0)
389 
390 /* create qp cfg2 */
391 #define ERDMA_CMD_CREATE_QP_TYPE_MASK GENMASK(3, 0)
392 
393 /* create qp cqn_mtt_cfg */
394 #define ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK GENMASK(31, 28)
395 #define ERDMA_CMD_CREATE_QP_DB_CFG_MASK BIT(25)
396 #define ERDMA_CMD_CREATE_QP_CQN_MASK GENMASK(23, 0)
397 
398 /* create qp mtt_cfg */
399 #define ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK GENMASK(31, 12)
400 #define ERDMA_CMD_CREATE_QP_MTT_CNT_MASK GENMASK(11, 1)
401 #define ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK BIT(0)
402 
403 /* create qp db cfg */
404 #define ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK GENMASK(31, 16)
405 #define ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK GENMASK(15, 0)
406 
407 #define ERDMA_CMDQ_CREATE_QP_RESP_COOKIE_MASK GENMASK_ULL(31, 0)
408 
409 struct erdma_cmdq_create_qp_req {
410 	u64 hdr;
411 	u32 cfg0;
412 	u32 cfg1;
413 	u32 sq_cqn_mtt_cfg;
414 	u32 rq_cqn_mtt_cfg;
415 	u64 sq_buf_addr;
416 	u64 rq_buf_addr;
417 	u32 sq_mtt_cfg;
418 	u32 rq_mtt_cfg;
419 	u64 sq_dbrec_dma;
420 	u64 rq_dbrec_dma;
421 
422 	u64 sq_mtt_entry[3];
423 	u64 rq_mtt_entry[3];
424 
425 	u32 db_cfg;
426 	u32 cfg2;
427 };
428 
429 struct erdma_cmdq_destroy_qp_req {
430 	u64 hdr;
431 	u32 qpn;
432 };
433 
434 struct erdma_cmdq_reflush_req {
435 	u64 hdr;
436 	u32 qpn;
437 	u32 sq_pi;
438 	u32 rq_pi;
439 };
440 
441 #define ERDMA_HW_RESP_SIZE 256
442 
443 struct erdma_cmdq_query_req {
444 	u64 hdr;
445 	u32 rsvd;
446 	u32 index;
447 
448 	u64 target_addr;
449 	u32 target_length;
450 };
451 
452 #define ERDMA_HW_RESP_MAGIC 0x5566
453 
454 struct erdma_cmdq_query_resp_hdr {
455 	u16 magic;
456 	u8 ver;
457 	u8 length;
458 
459 	u32 index;
460 	u32 rsvd[2];
461 };
462 
463 struct erdma_cmdq_query_stats_resp {
464 	struct erdma_cmdq_query_resp_hdr hdr;
465 
466 	u64 tx_req_cnt;
467 	u64 tx_packets_cnt;
468 	u64 tx_bytes_cnt;
469 	u64 tx_drop_packets_cnt;
470 	u64 tx_bps_meter_drop_packets_cnt;
471 	u64 tx_pps_meter_drop_packets_cnt;
472 	u64 rx_packets_cnt;
473 	u64 rx_bytes_cnt;
474 	u64 rx_drop_packets_cnt;
475 	u64 rx_bps_meter_drop_packets_cnt;
476 	u64 rx_pps_meter_drop_packets_cnt;
477 };
478 
479 enum erdma_network_type {
480 	ERDMA_NETWORK_TYPE_IPV4 = 0,
481 	ERDMA_NETWORK_TYPE_IPV6 = 1,
482 };
483 
484 enum erdma_set_gid_op {
485 	ERDMA_SET_GID_OP_ADD = 0,
486 	ERDMA_SET_GID_OP_DEL = 1,
487 };
488 
489 /* set gid cfg */
490 #define ERDMA_CMD_SET_GID_SGID_IDX_MASK GENMASK(15, 0)
491 #define ERDMA_CMD_SET_GID_NTYPE_MASK BIT(16)
492 #define ERDMA_CMD_SET_GID_OP_MASK BIT(31)
493 
494 struct erdma_cmdq_set_gid_req {
495 	u64 hdr;
496 	u32 cfg;
497 	u8 gid[ERDMA_ROCEV2_GID_SIZE];
498 };
499 
500 /* cap qword 0 definition */
501 #define ERDMA_CMD_DEV_CAP_MAX_GID_MASK GENMASK_ULL(51, 48)
502 #define ERDMA_CMD_DEV_CAP_MAX_CQE_MASK GENMASK_ULL(47, 40)
503 #define ERDMA_CMD_DEV_CAP_FLAGS_MASK GENMASK_ULL(31, 24)
504 #define ERDMA_CMD_DEV_CAP_MAX_RECV_WR_MASK GENMASK_ULL(23, 16)
505 #define ERDMA_CMD_DEV_CAP_MAX_AH_MASK GENMASK_ULL(15, 8)
506 #define ERDMA_CMD_DEV_CAP_MAX_MR_SIZE_MASK GENMASK_ULL(7, 0)
507 
508 /* cap qword 1 definition */
509 #define ERDMA_CMD_DEV_CAP_DMA_LOCAL_KEY_MASK GENMASK_ULL(63, 32)
510 #define ERDMA_CMD_DEV_CAP_DEFAULT_CC_MASK GENMASK_ULL(31, 28)
511 #define ERDMA_CMD_DEV_CAP_QBLOCK_MASK GENMASK_ULL(27, 16)
512 #define ERDMA_CMD_DEV_CAP_MAX_MW_MASK GENMASK_ULL(7, 0)
513 
514 #define ERDMA_NQP_PER_QBLOCK 1024
515 
516 enum {
517 	ERDMA_DEV_CAP_FLAGS_ATOMIC = 1 << 7,
518 	ERDMA_DEV_CAP_FLAGS_MTT_VA = 1 << 5,
519 	ERDMA_DEV_CAP_FLAGS_EXTEND_DB = 1 << 3,
520 };
521 
522 #define ERDMA_CMD_INFO0_FW_VER_MASK GENMASK_ULL(31, 0)
523 
524 /* CQE hdr */
525 #define ERDMA_CQE_HDR_OWNER_MASK BIT(31)
526 #define ERDMA_CQE_HDR_OPCODE_MASK GENMASK(23, 16)
527 #define ERDMA_CQE_HDR_QTYPE_MASK GENMASK(15, 8)
528 #define ERDMA_CQE_HDR_SYNDROME_MASK GENMASK(7, 0)
529 
530 #define ERDMA_CQE_QTYPE_SQ 0
531 #define ERDMA_CQE_QTYPE_RQ 1
532 #define ERDMA_CQE_QTYPE_CMDQ 2
533 
534 #define ERDMA_CQE_NTYPE_MASK BIT(31)
535 #define ERDMA_CQE_SL_MASK GENMASK(27, 20)
536 #define ERDMA_CQE_SQPN_MASK GENMASK(19, 0)
537 
538 struct erdma_cqe {
539 	__be32 hdr;
540 	__be32 qe_idx;
541 	__be32 qpn;
542 	union {
543 		__le32 imm_data;
544 		__be32 inv_rkey;
545 	};
546 	__be32 size;
547 	union {
548 		struct {
549 			__be32 rsvd[3];
550 		} rc;
551 
552 		struct {
553 			__be32 rsvd[2];
554 			__be32 info;
555 		} ud;
556 	};
557 };
558 
559 struct erdma_sge {
560 	__aligned_le64 addr;
561 	__le32 length;
562 	__le32 key;
563 };
564 
565 /* Receive Queue Element */
566 struct erdma_rqe {
567 	__le16 qe_idx;
568 	__le16 rsvd0;
569 	__le32 qpn;
570 	__le32 rsvd1;
571 	__le32 rsvd2;
572 	__le64 to;
573 	__le32 length;
574 	__le32 stag;
575 };
576 
577 /* SQE */
578 #define ERDMA_SQE_HDR_SGL_LEN_MASK GENMASK_ULL(63, 56)
579 #define ERDMA_SQE_HDR_WQEBB_CNT_MASK GENMASK_ULL(54, 52)
580 #define ERDMA_SQE_HDR_QPN_MASK GENMASK_ULL(51, 32)
581 #define ERDMA_SQE_HDR_OPCODE_MASK GENMASK_ULL(31, 27)
582 #define ERDMA_SQE_HDR_DWQE_MASK BIT_ULL(26)
583 #define ERDMA_SQE_HDR_INLINE_MASK BIT_ULL(25)
584 #define ERDMA_SQE_HDR_FENCE_MASK BIT_ULL(24)
585 #define ERDMA_SQE_HDR_SE_MASK BIT_ULL(23)
586 #define ERDMA_SQE_HDR_CE_MASK BIT_ULL(22)
587 #define ERDMA_SQE_HDR_WQEBB_INDEX_MASK GENMASK_ULL(15, 0)
588 
589 /* REG MR attrs */
590 #define ERDMA_SQE_MR_ACCESS_MASK GENMASK(5, 1)
591 #define ERDMA_SQE_MR_MTT_TYPE_MASK GENMASK(7, 6)
592 #define ERDMA_SQE_MR_MTT_CNT_MASK GENMASK(31, 12)
593 
594 struct erdma_write_sqe {
595 	__le64 hdr;
596 	__be32 imm_data;
597 	__le32 length;
598 
599 	__le32 sink_stag;
600 	__le32 sink_to_l;
601 	__le32 sink_to_h;
602 
603 	__le32 rsvd;
604 
605 	struct erdma_sge sgl[];
606 };
607 
608 struct erdma_send_sqe_rc {
609 	__le64 hdr;
610 	union {
611 		__be32 imm_data;
612 		__le32 invalid_stag;
613 	};
614 
615 	__le32 length;
616 	struct erdma_sge sgl[];
617 };
618 
619 struct erdma_send_sqe_ud {
620 	__le64 hdr;
621 	__be32 imm_data;
622 	__le32 length;
623 	__le32 qkey;
624 	__le32 dst_qpn;
625 	__le32 ahn;
626 	__le32 rsvd;
627 	struct erdma_sge sgl[];
628 };
629 
630 struct erdma_readreq_sqe {
631 	__le64 hdr;
632 	__le32 invalid_stag;
633 	__le32 length;
634 	__le32 sink_stag;
635 	__le32 sink_to_l;
636 	__le32 sink_to_h;
637 	__le32 rsvd;
638 };
639 
640 struct erdma_atomic_sqe {
641 	__le64 hdr;
642 	__le64 rsvd;
643 	__le64 fetchadd_swap_data;
644 	__le64 cmp_data;
645 
646 	struct erdma_sge remote;
647 	struct erdma_sge sgl;
648 };
649 
650 struct erdma_reg_mr_sqe {
651 	__le64 hdr;
652 	__le64 addr;
653 	__le32 length;
654 	__le32 stag;
655 	__le32 attrs;
656 	__le32 rsvd;
657 };
658 
659 /* EQ related. */
660 #define ERDMA_DEFAULT_EQ_DEPTH 4096
661 
662 /* ceqe */
663 #define ERDMA_CEQE_HDR_DB_MASK BIT_ULL(63)
664 #define ERDMA_CEQE_HDR_PI_MASK GENMASK_ULL(55, 32)
665 #define ERDMA_CEQE_HDR_O_MASK BIT_ULL(31)
666 #define ERDMA_CEQE_HDR_CQN_MASK GENMASK_ULL(19, 0)
667 
668 /* aeqe */
669 #define ERDMA_AEQE_HDR_O_MASK BIT(31)
670 #define ERDMA_AEQE_HDR_TYPE_MASK GENMASK(23, 16)
671 #define ERDMA_AEQE_HDR_SUBTYPE_MASK GENMASK(7, 0)
672 
673 #define ERDMA_AE_TYPE_QP_FATAL_EVENT 0
674 #define ERDMA_AE_TYPE_QP_ERQ_ERR_EVENT 1
675 #define ERDMA_AE_TYPE_ACC_ERR_EVENT 2
676 #define ERDMA_AE_TYPE_CQ_ERR 3
677 #define ERDMA_AE_TYPE_OTHER_ERROR 4
678 
679 struct erdma_aeqe {
680 	__le32 hdr;
681 	__le32 event_data0;
682 	__le32 event_data1;
683 	__le32 rsvd;
684 };
685 
686 enum erdma_opcode {
687 	ERDMA_OP_WRITE = 0,
688 	ERDMA_OP_READ = 1,
689 	ERDMA_OP_SEND = 2,
690 	ERDMA_OP_SEND_WITH_IMM = 3,
691 
692 	ERDMA_OP_RECEIVE = 4,
693 	ERDMA_OP_RECV_IMM = 5,
694 	ERDMA_OP_RECV_INV = 6,
695 
696 	ERDMA_OP_RSVD0 = 7,
697 	ERDMA_OP_RSVD1 = 8,
698 	ERDMA_OP_WRITE_WITH_IMM = 9,
699 
700 	ERDMA_OP_RSVD2 = 10,
701 	ERDMA_OP_RSVD3 = 11,
702 
703 	ERDMA_OP_RSP_SEND_IMM = 12,
704 	ERDMA_OP_SEND_WITH_INV = 13,
705 
706 	ERDMA_OP_REG_MR = 14,
707 	ERDMA_OP_LOCAL_INV = 15,
708 	ERDMA_OP_READ_WITH_INV = 16,
709 	ERDMA_OP_ATOMIC_CAS = 17,
710 	ERDMA_OP_ATOMIC_FAA = 18,
711 	ERDMA_NUM_OPCODES = 19,
712 	ERDMA_OP_INVALID = ERDMA_NUM_OPCODES + 1
713 };
714 
715 enum erdma_wc_status {
716 	ERDMA_WC_SUCCESS = 0,
717 	ERDMA_WC_GENERAL_ERR = 1,
718 	ERDMA_WC_RECV_WQE_FORMAT_ERR = 2,
719 	ERDMA_WC_RECV_STAG_INVALID_ERR = 3,
720 	ERDMA_WC_RECV_ADDR_VIOLATION_ERR = 4,
721 	ERDMA_WC_RECV_RIGHT_VIOLATION_ERR = 5,
722 	ERDMA_WC_RECV_PDID_ERR = 6,
723 	ERDMA_WC_RECV_WARRPING_ERR = 7,
724 	ERDMA_WC_SEND_WQE_FORMAT_ERR = 8,
725 	ERDMA_WC_SEND_WQE_ORD_EXCEED = 9,
726 	ERDMA_WC_SEND_STAG_INVALID_ERR = 10,
727 	ERDMA_WC_SEND_ADDR_VIOLATION_ERR = 11,
728 	ERDMA_WC_SEND_RIGHT_VIOLATION_ERR = 12,
729 	ERDMA_WC_SEND_PDID_ERR = 13,
730 	ERDMA_WC_SEND_WARRPING_ERR = 14,
731 	ERDMA_WC_FLUSH_ERR = 15,
732 	ERDMA_WC_RETRY_EXC_ERR = 16,
733 	ERDMA_NUM_WC_STATUS
734 };
735 
736 enum erdma_vendor_err {
737 	ERDMA_WC_VENDOR_NO_ERR = 0,
738 	ERDMA_WC_VENDOR_INVALID_RQE = 1,
739 	ERDMA_WC_VENDOR_RQE_INVALID_STAG = 2,
740 	ERDMA_WC_VENDOR_RQE_ADDR_VIOLATION = 3,
741 	ERDMA_WC_VENDOR_RQE_ACCESS_RIGHT_ERR = 4,
742 	ERDMA_WC_VENDOR_RQE_INVALID_PD = 5,
743 	ERDMA_WC_VENDOR_RQE_WRAP_ERR = 6,
744 	ERDMA_WC_VENDOR_INVALID_SQE = 0x20,
745 	ERDMA_WC_VENDOR_ZERO_ORD = 0x21,
746 	ERDMA_WC_VENDOR_SQE_INVALID_STAG = 0x30,
747 	ERDMA_WC_VENDOR_SQE_ADDR_VIOLATION = 0x31,
748 	ERDMA_WC_VENDOR_SQE_ACCESS_ERR = 0x32,
749 	ERDMA_WC_VENDOR_SQE_INVALID_PD = 0x33,
750 	ERDMA_WC_VENDOR_SQE_WARP_ERR = 0x34
751 };
752 
753 #endif
754