1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell IPSEC offload driver
3 *
4 * Copyright (C) 2024 Marvell.
5 */
6
7 #include <net/xfrm.h>
8 #include <linux/netdevice.h>
9 #include <linux/bitfield.h>
10 #include <crypto/aead.h>
11 #include <crypto/gcm.h>
12
13 #include "otx2_common.h"
14 #include "otx2_struct.h"
15 #include "cn10k_ipsec.h"
16
is_dev_support_ipsec_offload(struct pci_dev * pdev)17 static bool is_dev_support_ipsec_offload(struct pci_dev *pdev)
18 {
19 return is_dev_cn10ka_b0(pdev) || is_dev_cn10kb(pdev);
20 }
21
cn10k_cpt_device_set_inuse(struct otx2_nic * pf)22 static bool cn10k_cpt_device_set_inuse(struct otx2_nic *pf)
23 {
24 enum cn10k_cpt_hw_state_e state;
25
26 while (true) {
27 state = atomic_cmpxchg(&pf->ipsec.cpt_state,
28 CN10K_CPT_HW_AVAILABLE,
29 CN10K_CPT_HW_IN_USE);
30 if (state == CN10K_CPT_HW_AVAILABLE)
31 return true;
32 if (state == CN10K_CPT_HW_UNAVAILABLE)
33 return false;
34
35 mdelay(1);
36 }
37 }
38
cn10k_cpt_device_set_available(struct otx2_nic * pf)39 static void cn10k_cpt_device_set_available(struct otx2_nic *pf)
40 {
41 atomic_set(&pf->ipsec.cpt_state, CN10K_CPT_HW_AVAILABLE);
42 }
43
cn10k_cpt_device_set_unavailable(struct otx2_nic * pf)44 static void cn10k_cpt_device_set_unavailable(struct otx2_nic *pf)
45 {
46 atomic_set(&pf->ipsec.cpt_state, CN10K_CPT_HW_UNAVAILABLE);
47 }
48
cn10k_outb_cptlf_attach(struct otx2_nic * pf)49 static int cn10k_outb_cptlf_attach(struct otx2_nic *pf)
50 {
51 struct rsrc_attach *attach;
52 int ret = -ENOMEM;
53
54 mutex_lock(&pf->mbox.lock);
55 /* Get memory to put this msg */
56 attach = otx2_mbox_alloc_msg_attach_resources(&pf->mbox);
57 if (!attach)
58 goto unlock;
59
60 attach->cptlfs = true;
61 attach->modify = true;
62
63 /* Send attach request to AF */
64 ret = otx2_sync_mbox_msg(&pf->mbox);
65
66 unlock:
67 mutex_unlock(&pf->mbox.lock);
68 return ret;
69 }
70
cn10k_outb_cptlf_detach(struct otx2_nic * pf)71 static int cn10k_outb_cptlf_detach(struct otx2_nic *pf)
72 {
73 struct rsrc_detach *detach;
74 int ret = -ENOMEM;
75
76 mutex_lock(&pf->mbox.lock);
77 detach = otx2_mbox_alloc_msg_detach_resources(&pf->mbox);
78 if (!detach)
79 goto unlock;
80
81 detach->partial = true;
82 detach->cptlfs = true;
83
84 /* Send detach request to AF */
85 ret = otx2_sync_mbox_msg(&pf->mbox);
86
87 unlock:
88 mutex_unlock(&pf->mbox.lock);
89 return ret;
90 }
91
cn10k_outb_cptlf_alloc(struct otx2_nic * pf)92 static int cn10k_outb_cptlf_alloc(struct otx2_nic *pf)
93 {
94 struct cpt_lf_alloc_req_msg *req;
95 int ret = -ENOMEM;
96
97 mutex_lock(&pf->mbox.lock);
98 req = otx2_mbox_alloc_msg_cpt_lf_alloc(&pf->mbox);
99 if (!req)
100 goto unlock;
101
102 /* PF function */
103 req->nix_pf_func = pf->pcifunc;
104 /* Enable SE-IE Engine Group */
105 req->eng_grpmsk = 1 << CN10K_DEF_CPT_IPSEC_EGRP;
106
107 ret = otx2_sync_mbox_msg(&pf->mbox);
108
109 unlock:
110 mutex_unlock(&pf->mbox.lock);
111 return ret;
112 }
113
cn10k_outb_cptlf_free(struct otx2_nic * pf)114 static void cn10k_outb_cptlf_free(struct otx2_nic *pf)
115 {
116 mutex_lock(&pf->mbox.lock);
117 otx2_mbox_alloc_msg_cpt_lf_free(&pf->mbox);
118 otx2_sync_mbox_msg(&pf->mbox);
119 mutex_unlock(&pf->mbox.lock);
120 }
121
cn10k_outb_cptlf_config(struct otx2_nic * pf)122 static int cn10k_outb_cptlf_config(struct otx2_nic *pf)
123 {
124 struct cpt_inline_ipsec_cfg_msg *req;
125 int ret = -ENOMEM;
126
127 mutex_lock(&pf->mbox.lock);
128 req = otx2_mbox_alloc_msg_cpt_inline_ipsec_cfg(&pf->mbox);
129 if (!req)
130 goto unlock;
131
132 req->dir = CPT_INLINE_OUTBOUND;
133 req->enable = 1;
134 req->nix_pf_func = pf->pcifunc;
135 ret = otx2_sync_mbox_msg(&pf->mbox);
136 unlock:
137 mutex_unlock(&pf->mbox.lock);
138 return ret;
139 }
140
cn10k_outb_cptlf_iq_enable(struct otx2_nic * pf)141 static void cn10k_outb_cptlf_iq_enable(struct otx2_nic *pf)
142 {
143 u64 reg_val;
144
145 /* Set Execution Enable of instruction queue */
146 reg_val = otx2_read64(pf, CN10K_CPT_LF_INPROG);
147 reg_val |= BIT_ULL(16);
148 otx2_write64(pf, CN10K_CPT_LF_INPROG, reg_val);
149
150 /* Set iqueue's enqueuing */
151 reg_val = otx2_read64(pf, CN10K_CPT_LF_CTL);
152 reg_val |= BIT_ULL(0);
153 otx2_write64(pf, CN10K_CPT_LF_CTL, reg_val);
154 }
155
cn10k_outb_cptlf_iq_disable(struct otx2_nic * pf)156 static void cn10k_outb_cptlf_iq_disable(struct otx2_nic *pf)
157 {
158 u32 inflight, grb_cnt, gwb_cnt;
159 u32 nq_ptr, dq_ptr;
160 int timeout = 20;
161 u64 reg_val;
162 int cnt;
163
164 /* Disable instructions enqueuing */
165 otx2_write64(pf, CN10K_CPT_LF_CTL, 0ull);
166
167 /* Wait for instruction queue to become empty.
168 * CPT_LF_INPROG.INFLIGHT count is zero
169 */
170 do {
171 reg_val = otx2_read64(pf, CN10K_CPT_LF_INPROG);
172 inflight = FIELD_GET(CPT_LF_INPROG_INFLIGHT, reg_val);
173 if (!inflight)
174 break;
175
176 usleep_range(10000, 20000);
177 if (timeout-- < 0) {
178 netdev_err(pf->netdev, "Timeout to cleanup CPT IQ\n");
179 break;
180 }
181 } while (1);
182
183 /* Disable executions in the LF's queue,
184 * the queue should be empty at this point
185 */
186 reg_val &= ~BIT_ULL(16);
187 otx2_write64(pf, CN10K_CPT_LF_INPROG, reg_val);
188
189 /* Wait for instruction queue to become empty */
190 cnt = 0;
191 do {
192 reg_val = otx2_read64(pf, CN10K_CPT_LF_INPROG);
193 if (reg_val & BIT_ULL(31))
194 cnt = 0;
195 else
196 cnt++;
197 reg_val = otx2_read64(pf, CN10K_CPT_LF_Q_GRP_PTR);
198 nq_ptr = FIELD_GET(CPT_LF_Q_GRP_PTR_DQ_PTR, reg_val);
199 dq_ptr = FIELD_GET(CPT_LF_Q_GRP_PTR_DQ_PTR, reg_val);
200 } while ((cnt < 10) && (nq_ptr != dq_ptr));
201
202 cnt = 0;
203 do {
204 reg_val = otx2_read64(pf, CN10K_CPT_LF_INPROG);
205 inflight = FIELD_GET(CPT_LF_INPROG_INFLIGHT, reg_val);
206 grb_cnt = FIELD_GET(CPT_LF_INPROG_GRB_CNT, reg_val);
207 gwb_cnt = FIELD_GET(CPT_LF_INPROG_GWB_CNT, reg_val);
208 if (inflight == 0 && gwb_cnt < 40 &&
209 (grb_cnt == 0 || grb_cnt == 40))
210 cnt++;
211 else
212 cnt = 0;
213 } while (cnt < 10);
214 }
215
216 /* Allocate memory for CPT outbound Instruction queue.
217 * Instruction queue memory format is:
218 * -----------------------------
219 * | Instruction Group memory |
220 * | (CPT_LF_Q_SIZE[SIZE_DIV40] |
221 * | x 16 Bytes) |
222 * | |
223 * ----------------------------- <-- CPT_LF_Q_BASE[ADDR]
224 * | Flow Control (128 Bytes) |
225 * | |
226 * -----------------------------
227 * | Instruction Memory |
228 * | (CPT_LF_Q_SIZE[SIZE_DIV40] |
229 * | × 40 × 64 bytes) |
230 * | |
231 * -----------------------------
232 */
cn10k_outb_cptlf_iq_alloc(struct otx2_nic * pf)233 static int cn10k_outb_cptlf_iq_alloc(struct otx2_nic *pf)
234 {
235 struct cn10k_cpt_inst_queue *iq = &pf->ipsec.iq;
236
237 iq->size = CN10K_CPT_INST_QLEN_BYTES + CN10K_CPT_Q_FC_LEN +
238 CN10K_CPT_INST_GRP_QLEN_BYTES + OTX2_ALIGN;
239
240 iq->real_vaddr = dma_alloc_coherent(pf->dev, iq->size,
241 &iq->real_dma_addr, GFP_KERNEL);
242 if (!iq->real_vaddr)
243 return -ENOMEM;
244
245 /* iq->vaddr/dma_addr points to Flow Control location */
246 iq->vaddr = iq->real_vaddr + CN10K_CPT_INST_GRP_QLEN_BYTES;
247 iq->dma_addr = iq->real_dma_addr + CN10K_CPT_INST_GRP_QLEN_BYTES;
248
249 /* Align pointers */
250 iq->vaddr = PTR_ALIGN(iq->vaddr, OTX2_ALIGN);
251 iq->dma_addr = PTR_ALIGN(iq->dma_addr, OTX2_ALIGN);
252 return 0;
253 }
254
cn10k_outb_cptlf_iq_free(struct otx2_nic * pf)255 static void cn10k_outb_cptlf_iq_free(struct otx2_nic *pf)
256 {
257 struct cn10k_cpt_inst_queue *iq = &pf->ipsec.iq;
258
259 if (iq->real_vaddr)
260 dma_free_coherent(pf->dev, iq->size, iq->real_vaddr,
261 iq->real_dma_addr);
262
263 iq->real_vaddr = NULL;
264 iq->vaddr = NULL;
265 }
266
cn10k_outb_cptlf_iq_init(struct otx2_nic * pf)267 static int cn10k_outb_cptlf_iq_init(struct otx2_nic *pf)
268 {
269 u64 reg_val;
270 int ret;
271
272 /* Allocate Memory for CPT IQ */
273 ret = cn10k_outb_cptlf_iq_alloc(pf);
274 if (ret)
275 return ret;
276
277 /* Disable IQ */
278 cn10k_outb_cptlf_iq_disable(pf);
279
280 /* Set IQ base address */
281 otx2_write64(pf, CN10K_CPT_LF_Q_BASE, pf->ipsec.iq.dma_addr);
282
283 /* Set IQ size */
284 reg_val = FIELD_PREP(CPT_LF_Q_SIZE_DIV40, CN10K_CPT_SIZE_DIV40 +
285 CN10K_CPT_EXTRA_SIZE_DIV40);
286 otx2_write64(pf, CN10K_CPT_LF_Q_SIZE, reg_val);
287
288 return 0;
289 }
290
cn10k_outb_cptlf_init(struct otx2_nic * pf)291 static int cn10k_outb_cptlf_init(struct otx2_nic *pf)
292 {
293 int ret;
294
295 /* Initialize CPTLF Instruction Queue (IQ) */
296 ret = cn10k_outb_cptlf_iq_init(pf);
297 if (ret)
298 return ret;
299
300 /* Configure CPTLF for outbound ipsec offload */
301 ret = cn10k_outb_cptlf_config(pf);
302 if (ret)
303 goto iq_clean;
304
305 /* Enable CPTLF IQ */
306 cn10k_outb_cptlf_iq_enable(pf);
307 return 0;
308 iq_clean:
309 cn10k_outb_cptlf_iq_free(pf);
310 return ret;
311 }
312
cn10k_outb_cpt_init(struct net_device * netdev)313 static int cn10k_outb_cpt_init(struct net_device *netdev)
314 {
315 struct otx2_nic *pf = netdev_priv(netdev);
316 int ret;
317
318 /* Attach a CPT LF for outbound ipsec offload */
319 ret = cn10k_outb_cptlf_attach(pf);
320 if (ret)
321 return ret;
322
323 /* Allocate a CPT LF for outbound ipsec offload */
324 ret = cn10k_outb_cptlf_alloc(pf);
325 if (ret)
326 goto detach;
327
328 /* Initialize the CPTLF for outbound ipsec offload */
329 ret = cn10k_outb_cptlf_init(pf);
330 if (ret)
331 goto lf_free;
332
333 pf->ipsec.io_addr = (__force u64)otx2_get_regaddr(pf,
334 CN10K_CPT_LF_NQX(0));
335
336 /* Set ipsec offload enabled for this device */
337 pf->flags |= OTX2_FLAG_IPSEC_OFFLOAD_ENABLED;
338
339 cn10k_cpt_device_set_available(pf);
340 return 0;
341
342 lf_free:
343 cn10k_outb_cptlf_free(pf);
344 detach:
345 cn10k_outb_cptlf_detach(pf);
346 return ret;
347 }
348
cn10k_outb_cpt_clean(struct otx2_nic * pf)349 static int cn10k_outb_cpt_clean(struct otx2_nic *pf)
350 {
351 int ret;
352
353 if (!cn10k_cpt_device_set_inuse(pf)) {
354 netdev_err(pf->netdev, "CPT LF device unavailable\n");
355 return -ENODEV;
356 }
357
358 /* Set ipsec offload disabled for this device */
359 pf->flags &= ~OTX2_FLAG_IPSEC_OFFLOAD_ENABLED;
360
361 /* Disable CPTLF Instruction Queue (IQ) */
362 cn10k_outb_cptlf_iq_disable(pf);
363
364 /* Set IQ base address and size to 0 */
365 otx2_write64(pf, CN10K_CPT_LF_Q_BASE, 0);
366 otx2_write64(pf, CN10K_CPT_LF_Q_SIZE, 0);
367
368 /* Free CPTLF IQ */
369 cn10k_outb_cptlf_iq_free(pf);
370
371 /* Free and detach CPT LF */
372 cn10k_outb_cptlf_free(pf);
373 ret = cn10k_outb_cptlf_detach(pf);
374 if (ret)
375 netdev_err(pf->netdev, "Failed to detach CPT LF\n");
376
377 cn10k_cpt_device_set_unavailable(pf);
378 return ret;
379 }
380
cn10k_cpt_inst_flush(struct otx2_nic * pf,struct cpt_inst_s * inst,u64 size)381 static void cn10k_cpt_inst_flush(struct otx2_nic *pf, struct cpt_inst_s *inst,
382 u64 size)
383 {
384 struct otx2_lmt_info *lmt_info;
385 u64 val = 0, tar_addr = 0;
386
387 lmt_info = per_cpu_ptr(pf->hw.lmt_info, smp_processor_id());
388 /* FIXME: val[0:10] LMT_ID.
389 * [12:15] no of LMTST - 1 in the burst.
390 * [19:63] data size of each LMTST in the burst except first.
391 */
392 val = (lmt_info->lmt_id & 0x7FF);
393 /* Target address for LMTST flush tells HW how many 128bit
394 * words are present.
395 * tar_addr[6:4] size of first LMTST - 1 in units of 128b.
396 */
397 tar_addr |= pf->ipsec.io_addr | (((size / 16) - 1) & 0x7) << 4;
398 dma_wmb();
399 memcpy((u64 *)lmt_info->lmt_addr, inst, size);
400 cn10k_lmt_flush(val, tar_addr);
401 }
402
cn10k_wait_for_cpt_respose(struct otx2_nic * pf,struct cpt_res_s * res)403 static int cn10k_wait_for_cpt_respose(struct otx2_nic *pf,
404 struct cpt_res_s *res)
405 {
406 unsigned long timeout = jiffies + msecs_to_jiffies(100);
407 u64 *completion_ptr = (u64 *)res;
408
409 do {
410 if (time_after(jiffies, timeout)) {
411 netdev_err(pf->netdev, "CPT response timeout\n");
412 return -EBUSY;
413 }
414 } while ((READ_ONCE(*completion_ptr) & CN10K_CPT_COMP_E_MASK) ==
415 CN10K_CPT_COMP_E_NOTDONE);
416
417 if (!(res->compcode == CN10K_CPT_COMP_E_GOOD ||
418 res->compcode == CN10K_CPT_COMP_E_WARN) || res->uc_compcode) {
419 netdev_err(pf->netdev, "compcode=%x doneint=%x\n",
420 res->compcode, res->doneint);
421 netdev_err(pf->netdev, "uc_compcode=%x uc_info=%llx esn=%llx\n",
422 res->uc_compcode, (u64)res->uc_info, res->esn);
423 }
424 return 0;
425 }
426
cn10k_outb_write_sa(struct otx2_nic * pf,struct qmem * sa_info)427 static int cn10k_outb_write_sa(struct otx2_nic *pf, struct qmem *sa_info)
428 {
429 dma_addr_t res_iova, dptr_iova, sa_iova;
430 struct cn10k_tx_sa_s *sa_dptr;
431 struct cpt_inst_s inst = {};
432 struct cpt_res_s *res;
433 u32 sa_size, off;
434 u64 *sptr, *dptr;
435 u64 reg_val;
436 int ret;
437
438 sa_iova = sa_info->iova;
439 if (!sa_iova)
440 return -EINVAL;
441
442 res = dma_alloc_coherent(pf->dev, sizeof(struct cpt_res_s),
443 &res_iova, GFP_ATOMIC);
444 if (!res)
445 return -ENOMEM;
446
447 sa_size = sizeof(struct cn10k_tx_sa_s);
448 sa_dptr = dma_alloc_coherent(pf->dev, sa_size, &dptr_iova, GFP_ATOMIC);
449 if (!sa_dptr) {
450 dma_free_coherent(pf->dev, sizeof(struct cpt_res_s), res,
451 res_iova);
452 return -ENOMEM;
453 }
454
455 sptr = (__force u64 *)sa_info->base;
456 dptr = (__force u64 *)sa_dptr;
457 for (off = 0; off < (sa_size / 8); off++)
458 *(dptr + off) = (__force u64)cpu_to_be64(*(sptr + off));
459
460 res->compcode = CN10K_CPT_COMP_E_NOTDONE;
461 inst.res_addr = res_iova;
462 inst.dptr = (u64)dptr_iova;
463 inst.param2 = sa_size >> 3;
464 inst.dlen = sa_size;
465 inst.opcode_major = CN10K_IPSEC_MAJOR_OP_WRITE_SA;
466 inst.opcode_minor = CN10K_IPSEC_MINOR_OP_WRITE_SA;
467 inst.cptr = sa_iova;
468 inst.ctx_val = 1;
469 inst.egrp = CN10K_DEF_CPT_IPSEC_EGRP;
470
471 /* Check if CPT-LF available */
472 if (!cn10k_cpt_device_set_inuse(pf)) {
473 ret = -ENODEV;
474 goto free_mem;
475 }
476
477 cn10k_cpt_inst_flush(pf, &inst, sizeof(struct cpt_inst_s));
478 dma_wmb();
479 ret = cn10k_wait_for_cpt_respose(pf, res);
480 if (ret)
481 goto set_available;
482
483 /* Trigger CTX flush to write dirty data back to DRAM */
484 reg_val = FIELD_PREP(CPT_LF_CTX_FLUSH, sa_iova >> 7);
485 otx2_write64(pf, CN10K_CPT_LF_CTX_FLUSH, reg_val);
486
487 set_available:
488 cn10k_cpt_device_set_available(pf);
489 free_mem:
490 dma_free_coherent(pf->dev, sa_size, sa_dptr, dptr_iova);
491 dma_free_coherent(pf->dev, sizeof(struct cpt_res_s), res, res_iova);
492 return ret;
493 }
494
cn10k_ipsec_get_hw_ctx_offset(void)495 static int cn10k_ipsec_get_hw_ctx_offset(void)
496 {
497 /* Offset on Hardware-context offset in word */
498 return (offsetof(struct cn10k_tx_sa_s, hw_ctx) / sizeof(u64)) & 0x7F;
499 }
500
cn10k_ipsec_get_ctx_push_size(void)501 static int cn10k_ipsec_get_ctx_push_size(void)
502 {
503 /* Context push size is round up and in multiple of 8 Byte */
504 return (roundup(offsetof(struct cn10k_tx_sa_s, hw_ctx), 8) / 8) & 0x7F;
505 }
506
cn10k_ipsec_get_aes_key_len(int key_len)507 static int cn10k_ipsec_get_aes_key_len(int key_len)
508 {
509 /* key_len is aes key length in bytes */
510 switch (key_len) {
511 case 16:
512 return CN10K_IPSEC_SA_AES_KEY_LEN_128;
513 case 24:
514 return CN10K_IPSEC_SA_AES_KEY_LEN_192;
515 default:
516 return CN10K_IPSEC_SA_AES_KEY_LEN_256;
517 }
518 }
519
cn10k_outb_prepare_sa(struct xfrm_state * x,struct cn10k_tx_sa_s * sa_entry)520 static void cn10k_outb_prepare_sa(struct xfrm_state *x,
521 struct cn10k_tx_sa_s *sa_entry)
522 {
523 int key_len = (x->aead->alg_key_len + 7) / 8;
524 struct net_device *netdev = x->xso.dev;
525 u8 *key = x->aead->alg_key;
526 struct otx2_nic *pf;
527 u32 *tmp_salt;
528 u64 *tmp_key;
529 int idx;
530
531 memset(sa_entry, 0, sizeof(struct cn10k_tx_sa_s));
532
533 /* context size, 128 Byte aligned up */
534 pf = netdev_priv(netdev);
535 sa_entry->ctx_size = (pf->ipsec.sa_size / OTX2_ALIGN) & 0xF;
536 sa_entry->hw_ctx_off = cn10k_ipsec_get_hw_ctx_offset();
537 sa_entry->ctx_push_size = cn10k_ipsec_get_ctx_push_size();
538
539 /* Ucode to skip two words of CPT_CTX_HW_S */
540 sa_entry->ctx_hdr_size = 1;
541
542 /* Allow Atomic operation (AOP) */
543 sa_entry->aop_valid = 1;
544
545 /* Outbound, ESP TRANSPORT/TUNNEL Mode, AES-GCM with */
546 sa_entry->sa_dir = CN10K_IPSEC_SA_DIR_OUTB;
547 sa_entry->ipsec_protocol = CN10K_IPSEC_SA_IPSEC_PROTO_ESP;
548 sa_entry->enc_type = CN10K_IPSEC_SA_ENCAP_TYPE_AES_GCM;
549 sa_entry->iv_src = CN10K_IPSEC_SA_IV_SRC_PACKET;
550 if (x->props.mode == XFRM_MODE_TUNNEL)
551 sa_entry->ipsec_mode = CN10K_IPSEC_SA_IPSEC_MODE_TUNNEL;
552 else
553 sa_entry->ipsec_mode = CN10K_IPSEC_SA_IPSEC_MODE_TRANSPORT;
554
555 /* Last 4 bytes are salt */
556 key_len -= 4;
557 sa_entry->aes_key_len = cn10k_ipsec_get_aes_key_len(key_len);
558 memcpy(sa_entry->cipher_key, key, key_len);
559 tmp_key = (u64 *)sa_entry->cipher_key;
560
561 for (idx = 0; idx < key_len / 8; idx++)
562 tmp_key[idx] = (__force u64)cpu_to_be64(tmp_key[idx]);
563
564 memcpy(&sa_entry->iv_gcm_salt, key + key_len, 4);
565 tmp_salt = (u32 *)&sa_entry->iv_gcm_salt;
566 *tmp_salt = (__force u32)cpu_to_be32(*tmp_salt);
567
568 /* Write SA context data to memory before enabling */
569 wmb();
570
571 /* Enable SA */
572 sa_entry->sa_valid = 1;
573 }
574
cn10k_ipsec_validate_state(struct xfrm_state * x,struct netlink_ext_ack * extack)575 static int cn10k_ipsec_validate_state(struct xfrm_state *x,
576 struct netlink_ext_ack *extack)
577 {
578 if (x->props.aalgo != SADB_AALG_NONE) {
579 NL_SET_ERR_MSG_MOD(extack,
580 "Cannot offload authenticated xfrm states");
581 return -EINVAL;
582 }
583 if (x->props.ealgo != SADB_X_EALG_AES_GCM_ICV16) {
584 NL_SET_ERR_MSG_MOD(extack,
585 "Only AES-GCM-ICV16 xfrm state may be offloaded");
586 return -EINVAL;
587 }
588 if (x->props.calgo != SADB_X_CALG_NONE) {
589 NL_SET_ERR_MSG_MOD(extack,
590 "Cannot offload compressed xfrm states");
591 return -EINVAL;
592 }
593 if (x->props.flags & XFRM_STATE_ESN) {
594 NL_SET_ERR_MSG_MOD(extack, "Cannot offload ESN xfrm states");
595 return -EINVAL;
596 }
597 if (x->props.family != AF_INET && x->props.family != AF_INET6) {
598 NL_SET_ERR_MSG_MOD(extack,
599 "Only IPv4/v6 xfrm states may be offloaded");
600 return -EINVAL;
601 }
602 if (x->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) {
603 NL_SET_ERR_MSG_MOD(extack,
604 "Cannot offload other than crypto-mode");
605 return -EINVAL;
606 }
607 if (x->props.mode != XFRM_MODE_TRANSPORT &&
608 x->props.mode != XFRM_MODE_TUNNEL) {
609 NL_SET_ERR_MSG_MOD(extack,
610 "Only tunnel/transport xfrm states may be offloaded");
611 return -EINVAL;
612 }
613 if (x->id.proto != IPPROTO_ESP) {
614 NL_SET_ERR_MSG_MOD(extack,
615 "Only ESP xfrm state may be offloaded");
616 return -EINVAL;
617 }
618 if (x->encap) {
619 NL_SET_ERR_MSG_MOD(extack,
620 "Encapsulated xfrm state may not be offloaded");
621 return -EINVAL;
622 }
623 if (!x->aead) {
624 NL_SET_ERR_MSG_MOD(extack,
625 "Cannot offload xfrm states without aead");
626 return -EINVAL;
627 }
628
629 if (x->aead->alg_icv_len != 128) {
630 NL_SET_ERR_MSG_MOD(extack,
631 "Cannot offload xfrm states with AEAD ICV length other than 128bit");
632 return -EINVAL;
633 }
634 if (x->aead->alg_key_len != 128 + 32 &&
635 x->aead->alg_key_len != 192 + 32 &&
636 x->aead->alg_key_len != 256 + 32) {
637 NL_SET_ERR_MSG_MOD(extack,
638 "Cannot offload xfrm states with AEAD key length other than 128/192/256bit");
639 return -EINVAL;
640 }
641 if (x->tfcpad) {
642 NL_SET_ERR_MSG_MOD(extack,
643 "Cannot offload xfrm states with tfc padding");
644 return -EINVAL;
645 }
646 if (!x->geniv) {
647 NL_SET_ERR_MSG_MOD(extack,
648 "Cannot offload xfrm states without geniv");
649 return -EINVAL;
650 }
651 if (strcmp(x->geniv, "seqiv")) {
652 NL_SET_ERR_MSG_MOD(extack,
653 "Cannot offload xfrm states with geniv other than seqiv");
654 return -EINVAL;
655 }
656 return 0;
657 }
658
cn10k_ipsec_inb_add_state(struct xfrm_state * x,struct netlink_ext_ack * extack)659 static int cn10k_ipsec_inb_add_state(struct xfrm_state *x,
660 struct netlink_ext_ack *extack)
661 {
662 NL_SET_ERR_MSG_MOD(extack, "xfrm inbound offload not supported");
663 return -EOPNOTSUPP;
664 }
665
cn10k_ipsec_outb_add_state(struct xfrm_state * x,struct netlink_ext_ack * extack)666 static int cn10k_ipsec_outb_add_state(struct xfrm_state *x,
667 struct netlink_ext_ack *extack)
668 {
669 struct net_device *netdev = x->xso.dev;
670 struct cn10k_tx_sa_s *sa_entry;
671 struct qmem *sa_info;
672 struct otx2_nic *pf;
673 int err;
674
675 err = cn10k_ipsec_validate_state(x, extack);
676 if (err)
677 return err;
678
679 pf = netdev_priv(netdev);
680
681 err = qmem_alloc(pf->dev, &sa_info, pf->ipsec.sa_size, OTX2_ALIGN);
682 if (err)
683 return err;
684
685 sa_entry = (struct cn10k_tx_sa_s *)sa_info->base;
686 cn10k_outb_prepare_sa(x, sa_entry);
687
688 err = cn10k_outb_write_sa(pf, sa_info);
689 if (err) {
690 NL_SET_ERR_MSG_MOD(extack, "Error writing outbound SA");
691 qmem_free(pf->dev, sa_info);
692 return err;
693 }
694
695 x->xso.offload_handle = (unsigned long)sa_info;
696 /* Enable static branch when first SA setup */
697 if (!pf->ipsec.outb_sa_count)
698 static_branch_enable(&cn10k_ipsec_sa_enabled);
699 pf->ipsec.outb_sa_count++;
700 return 0;
701 }
702
cn10k_ipsec_add_state(struct xfrm_state * x,struct netlink_ext_ack * extack)703 static int cn10k_ipsec_add_state(struct xfrm_state *x,
704 struct netlink_ext_ack *extack)
705 {
706 if (x->xso.dir == XFRM_DEV_OFFLOAD_IN)
707 return cn10k_ipsec_inb_add_state(x, extack);
708 else
709 return cn10k_ipsec_outb_add_state(x, extack);
710 }
711
cn10k_ipsec_del_state(struct xfrm_state * x)712 static void cn10k_ipsec_del_state(struct xfrm_state *x)
713 {
714 struct net_device *netdev = x->xso.dev;
715 struct cn10k_tx_sa_s *sa_entry;
716 struct qmem *sa_info;
717 struct otx2_nic *pf;
718 int err;
719
720 if (x->xso.dir == XFRM_DEV_OFFLOAD_IN)
721 return;
722
723 pf = netdev_priv(netdev);
724
725 sa_info = (struct qmem *)x->xso.offload_handle;
726 sa_entry = (struct cn10k_tx_sa_s *)sa_info->base;
727 memset(sa_entry, 0, sizeof(struct cn10k_tx_sa_s));
728 /* Disable SA in CPT h/w */
729 sa_entry->ctx_push_size = cn10k_ipsec_get_ctx_push_size();
730 sa_entry->ctx_size = (pf->ipsec.sa_size / OTX2_ALIGN) & 0xF;
731 sa_entry->aop_valid = 1;
732
733 err = cn10k_outb_write_sa(pf, sa_info);
734 if (err)
735 netdev_err(netdev, "Error (%d) deleting SA\n", err);
736
737 x->xso.offload_handle = 0;
738 qmem_free(pf->dev, sa_info);
739
740 /* If no more SA's then update netdev feature for potential change
741 * in NETIF_F_HW_ESP.
742 */
743 if (!--pf->ipsec.outb_sa_count)
744 queue_work(pf->ipsec.sa_workq, &pf->ipsec.sa_work);
745 }
746
cn10k_ipsec_offload_ok(struct sk_buff * skb,struct xfrm_state * x)747 static bool cn10k_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
748 {
749 if (x->props.family == AF_INET) {
750 /* Offload with IPv4 options is not supported yet */
751 if (ip_hdr(skb)->ihl > 5)
752 return false;
753 } else {
754 /* Offload with IPv6 extension headers is not support yet */
755 if (ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr))
756 return false;
757 }
758 return true;
759 }
760
761 static const struct xfrmdev_ops cn10k_ipsec_xfrmdev_ops = {
762 .xdo_dev_state_add = cn10k_ipsec_add_state,
763 .xdo_dev_state_delete = cn10k_ipsec_del_state,
764 .xdo_dev_offload_ok = cn10k_ipsec_offload_ok,
765 };
766
cn10k_ipsec_sa_wq_handler(struct work_struct * work)767 static void cn10k_ipsec_sa_wq_handler(struct work_struct *work)
768 {
769 struct cn10k_ipsec *ipsec = container_of(work, struct cn10k_ipsec,
770 sa_work);
771 struct otx2_nic *pf = container_of(ipsec, struct otx2_nic, ipsec);
772
773 /* Disable static branch when no more SA enabled */
774 static_branch_disable(&cn10k_ipsec_sa_enabled);
775 rtnl_lock();
776 netdev_update_features(pf->netdev);
777 rtnl_unlock();
778 }
779
cn10k_ipsec_ethtool_init(struct net_device * netdev,bool enable)780 int cn10k_ipsec_ethtool_init(struct net_device *netdev, bool enable)
781 {
782 struct otx2_nic *pf = netdev_priv(netdev);
783
784 /* IPsec offload supported on cn10k */
785 if (!is_dev_support_ipsec_offload(pf->pdev))
786 return -EOPNOTSUPP;
787
788 /* Initialize CPT for outbound ipsec offload */
789 if (enable)
790 return cn10k_outb_cpt_init(netdev);
791
792 /* Don't do CPT cleanup if SA installed */
793 if (pf->ipsec.outb_sa_count) {
794 netdev_err(pf->netdev, "SA installed on this device\n");
795 return -EBUSY;
796 }
797
798 return cn10k_outb_cpt_clean(pf);
799 }
800
cn10k_ipsec_init(struct net_device * netdev)801 int cn10k_ipsec_init(struct net_device *netdev)
802 {
803 struct otx2_nic *pf = netdev_priv(netdev);
804 u32 sa_size;
805
806 if (!is_dev_support_ipsec_offload(pf->pdev))
807 return 0;
808
809 /* Each SA entry size is 128 Byte round up in size */
810 sa_size = sizeof(struct cn10k_tx_sa_s) % OTX2_ALIGN ?
811 (sizeof(struct cn10k_tx_sa_s) / OTX2_ALIGN + 1) *
812 OTX2_ALIGN : sizeof(struct cn10k_tx_sa_s);
813 pf->ipsec.sa_size = sa_size;
814
815 INIT_WORK(&pf->ipsec.sa_work, cn10k_ipsec_sa_wq_handler);
816 pf->ipsec.sa_workq = alloc_workqueue("cn10k_ipsec_sa_workq", 0, 0);
817 if (!pf->ipsec.sa_workq) {
818 netdev_err(pf->netdev, "SA alloc workqueue failed\n");
819 return -ENOMEM;
820 }
821
822 /* Set xfrm device ops */
823 netdev->xfrmdev_ops = &cn10k_ipsec_xfrmdev_ops;
824 netdev->hw_features |= NETIF_F_HW_ESP;
825 netdev->hw_enc_features |= NETIF_F_HW_ESP;
826
827 cn10k_cpt_device_set_unavailable(pf);
828 return 0;
829 }
830 EXPORT_SYMBOL(cn10k_ipsec_init);
831
cn10k_ipsec_clean(struct otx2_nic * pf)832 void cn10k_ipsec_clean(struct otx2_nic *pf)
833 {
834 if (!is_dev_support_ipsec_offload(pf->pdev))
835 return;
836
837 if (!(pf->flags & OTX2_FLAG_IPSEC_OFFLOAD_ENABLED))
838 return;
839
840 if (pf->ipsec.sa_workq) {
841 destroy_workqueue(pf->ipsec.sa_workq);
842 pf->ipsec.sa_workq = NULL;
843 }
844
845 cn10k_outb_cpt_clean(pf);
846 }
847 EXPORT_SYMBOL(cn10k_ipsec_clean);
848
cn10k_ipsec_get_ip_data_len(struct xfrm_state * x,struct sk_buff * skb)849 static u16 cn10k_ipsec_get_ip_data_len(struct xfrm_state *x,
850 struct sk_buff *skb)
851 {
852 struct ipv6hdr *ipv6h;
853 struct iphdr *iph;
854 u8 *src;
855
856 src = (u8 *)skb->data + ETH_HLEN;
857
858 if (x->props.family == AF_INET) {
859 iph = (struct iphdr *)src;
860 return ntohs(iph->tot_len);
861 }
862
863 ipv6h = (struct ipv6hdr *)src;
864 return ntohs(ipv6h->payload_len) + sizeof(struct ipv6hdr);
865 }
866
867 /* Prepare CPT and NIX SQE scatter/gather subdescriptor structure.
868 * SG of NIX and CPT are same in size.
869 * Layout of a NIX SQE and CPT SG entry:
870 * -----------------------------
871 * | CPT Scatter Gather |
872 * | (SQE SIZE) |
873 * | |
874 * -----------------------------
875 * | NIX SQE |
876 * | (SQE SIZE) |
877 * | |
878 * -----------------------------
879 */
otx2_sqe_add_sg_ipsec(struct otx2_nic * pfvf,struct otx2_snd_queue * sq,struct sk_buff * skb,int num_segs,int * offset)880 bool otx2_sqe_add_sg_ipsec(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,
881 struct sk_buff *skb, int num_segs, int *offset)
882 {
883 struct cpt_sg_s *cpt_sg = NULL;
884 struct nix_sqe_sg_s *sg = NULL;
885 u64 dma_addr, *iova = NULL;
886 u64 *cpt_iova = NULL;
887 u16 *sg_lens = NULL;
888 int seg, len;
889
890 sq->sg[sq->head].num_segs = 0;
891 cpt_sg = (struct cpt_sg_s *)(sq->sqe_base - sq->sqe_size);
892
893 for (seg = 0; seg < num_segs; seg++) {
894 if ((seg % MAX_SEGS_PER_SG) == 0) {
895 sg = (struct nix_sqe_sg_s *)(sq->sqe_base + *offset);
896 sg->ld_type = NIX_SEND_LDTYPE_LDD;
897 sg->subdc = NIX_SUBDC_SG;
898 sg->segs = 0;
899 sg_lens = (void *)sg;
900 iova = (void *)sg + sizeof(*sg);
901 /* Next subdc always starts at a 16byte boundary.
902 * So if sg->segs is whether 2 or 3, offset += 16bytes.
903 */
904 if ((num_segs - seg) >= (MAX_SEGS_PER_SG - 1))
905 *offset += sizeof(*sg) + (3 * sizeof(u64));
906 else
907 *offset += sizeof(*sg) + sizeof(u64);
908
909 cpt_sg += (seg / MAX_SEGS_PER_SG) * 4;
910 cpt_iova = (void *)cpt_sg + sizeof(*cpt_sg);
911 }
912 dma_addr = otx2_dma_map_skb_frag(pfvf, skb, seg, &len);
913 if (dma_mapping_error(pfvf->dev, dma_addr))
914 return false;
915
916 sg_lens[seg % MAX_SEGS_PER_SG] = len;
917 sg->segs++;
918 *iova++ = dma_addr;
919 *cpt_iova++ = dma_addr;
920
921 /* Save DMA mapping info for later unmapping */
922 sq->sg[sq->head].dma_addr[seg] = dma_addr;
923 sq->sg[sq->head].size[seg] = len;
924 sq->sg[sq->head].num_segs++;
925
926 *cpt_sg = *(struct cpt_sg_s *)sg;
927 cpt_sg->rsvd_63_50 = 0;
928 }
929
930 sq->sg[sq->head].skb = (u64)skb;
931 return true;
932 }
933
cn10k_ipsec_get_param1(u8 iv_offset)934 static u16 cn10k_ipsec_get_param1(u8 iv_offset)
935 {
936 u16 param1_val;
937
938 /* Set Crypto mode, disable L3/L4 checksum */
939 param1_val = CN10K_IPSEC_INST_PARAM1_DIS_L4_CSUM |
940 CN10K_IPSEC_INST_PARAM1_DIS_L3_CSUM;
941 param1_val |= (u16)iv_offset << CN10K_IPSEC_INST_PARAM1_IV_OFFSET_SHIFT;
942 return param1_val;
943 }
944
cn10k_ipsec_transmit(struct otx2_nic * pf,struct netdev_queue * txq,struct otx2_snd_queue * sq,struct sk_buff * skb,int num_segs,int size)945 bool cn10k_ipsec_transmit(struct otx2_nic *pf, struct netdev_queue *txq,
946 struct otx2_snd_queue *sq, struct sk_buff *skb,
947 int num_segs, int size)
948 {
949 struct cpt_inst_s inst;
950 struct cpt_res_s *res;
951 struct xfrm_state *x;
952 struct qmem *sa_info;
953 dma_addr_t dptr_iova;
954 struct sec_path *sp;
955 u8 encap_offset;
956 u8 auth_offset;
957 u8 gthr_size;
958 u8 iv_offset;
959 u16 dlen;
960
961 /* Check for IPSEC offload enabled */
962 if (!(pf->flags & OTX2_FLAG_IPSEC_OFFLOAD_ENABLED))
963 goto drop;
964
965 sp = skb_sec_path(skb);
966 if (unlikely(!sp->len))
967 goto drop;
968
969 x = xfrm_input_state(skb);
970 if (unlikely(!x))
971 goto drop;
972
973 if (x->props.mode != XFRM_MODE_TRANSPORT &&
974 x->props.mode != XFRM_MODE_TUNNEL)
975 goto drop;
976
977 dlen = cn10k_ipsec_get_ip_data_len(x, skb);
978 if (dlen == 0 && netif_msg_tx_err(pf)) {
979 netdev_err(pf->netdev, "Invalid IP header, ip-length zero\n");
980 goto drop;
981 }
982
983 /* Check for valid SA context */
984 sa_info = (struct qmem *)x->xso.offload_handle;
985 if (!sa_info)
986 goto drop;
987
988 memset(&inst, 0, sizeof(struct cpt_inst_s));
989
990 /* Get authentication offset */
991 if (x->props.family == AF_INET)
992 auth_offset = sizeof(struct iphdr);
993 else
994 auth_offset = sizeof(struct ipv6hdr);
995
996 /* IV offset is after ESP header */
997 iv_offset = auth_offset + sizeof(struct ip_esp_hdr);
998 /* Encap will start after IV */
999 encap_offset = iv_offset + GCM_RFC4106_IV_SIZE;
1000
1001 /* CPT Instruction word-1 */
1002 res = (struct cpt_res_s *)(sq->cpt_resp->base + (64 * sq->head));
1003 res->compcode = 0;
1004 inst.res_addr = sq->cpt_resp->iova + (64 * sq->head);
1005
1006 /* CPT Instruction word-2 */
1007 inst.rvu_pf_func = pf->pcifunc;
1008
1009 /* CPT Instruction word-3:
1010 * Set QORD to force CPT_RES_S write completion
1011 */
1012 inst.qord = 1;
1013
1014 /* CPT Instruction word-4 */
1015 /* inst.dlen should not include ICV length */
1016 inst.dlen = dlen + ETH_HLEN - (x->aead->alg_icv_len / 8);
1017 inst.opcode_major = CN10K_IPSEC_MAJOR_OP_OUTB_IPSEC;
1018 inst.param1 = cn10k_ipsec_get_param1(iv_offset);
1019
1020 inst.param2 = encap_offset <<
1021 CN10K_IPSEC_INST_PARAM2_ENC_DATA_OFFSET_SHIFT;
1022 inst.param2 |= (u16)auth_offset <<
1023 CN10K_IPSEC_INST_PARAM2_AUTH_DATA_OFFSET_SHIFT;
1024
1025 /* CPT Instruction word-5 */
1026 gthr_size = num_segs / MAX_SEGS_PER_SG;
1027 gthr_size = (num_segs % MAX_SEGS_PER_SG) ? gthr_size + 1 : gthr_size;
1028
1029 gthr_size &= 0xF;
1030 dptr_iova = (sq->sqe_ring->iova + (sq->head * (sq->sqe_size * 2)));
1031 inst.dptr = dptr_iova | ((u64)gthr_size << 60);
1032
1033 /* CPT Instruction word-6 */
1034 inst.rptr = inst.dptr;
1035
1036 /* CPT Instruction word-7 */
1037 inst.cptr = sa_info->iova;
1038 inst.ctx_val = 1;
1039 inst.egrp = CN10K_DEF_CPT_IPSEC_EGRP;
1040
1041 /* CPT Instruction word-0 */
1042 inst.nixtxl = (size / 16) - 1;
1043 inst.dat_offset = ETH_HLEN;
1044 inst.nixtx_offset = sq->sqe_size;
1045
1046 netdev_tx_sent_queue(txq, skb->len);
1047
1048 /* Finally Flush the CPT instruction */
1049 sq->head++;
1050 sq->head &= (sq->sqe_cnt - 1);
1051 cn10k_cpt_inst_flush(pf, &inst, sizeof(struct cpt_inst_s));
1052 return true;
1053 drop:
1054 dev_kfree_skb_any(skb);
1055 return false;
1056 }
1057