1 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
2 /* Google virtual Ethernet (gve) driver
3 *
4 * Copyright (C) 2015-2021 Google, Inc.
5 */
6
7 #include "gve.h"
8 #include "gve_dqo.h"
9 #include "gve_adminq.h"
10 #include "gve_utils.h"
11 #include <linux/ip.h>
12 #include <linux/ipv6.h>
13 #include <linux/skbuff.h>
14 #include <linux/slab.h>
15 #include <net/ip6_checksum.h>
16 #include <net/ipv6.h>
17 #include <net/tcp.h>
18
gve_rx_free_hdr_bufs(struct gve_priv * priv,struct gve_rx_ring * rx)19 static void gve_rx_free_hdr_bufs(struct gve_priv *priv, struct gve_rx_ring *rx)
20 {
21 struct device *hdev = &priv->pdev->dev;
22 int buf_count = rx->dqo.bufq.mask + 1;
23
24 if (rx->dqo.hdr_bufs.data) {
25 dma_free_coherent(hdev, priv->header_buf_size * buf_count,
26 rx->dqo.hdr_bufs.data, rx->dqo.hdr_bufs.addr);
27 rx->dqo.hdr_bufs.data = NULL;
28 }
29 }
30
gve_rx_init_ring_state_dqo(struct gve_rx_ring * rx,const u32 buffer_queue_slots,const u32 completion_queue_slots)31 static void gve_rx_init_ring_state_dqo(struct gve_rx_ring *rx,
32 const u32 buffer_queue_slots,
33 const u32 completion_queue_slots)
34 {
35 int i;
36
37 /* Set buffer queue state */
38 rx->dqo.bufq.mask = buffer_queue_slots - 1;
39 rx->dqo.bufq.head = 0;
40 rx->dqo.bufq.tail = 0;
41
42 /* Set completion queue state */
43 rx->dqo.complq.num_free_slots = completion_queue_slots;
44 rx->dqo.complq.mask = completion_queue_slots - 1;
45 rx->dqo.complq.cur_gen_bit = 0;
46 rx->dqo.complq.head = 0;
47
48 /* Set RX SKB context */
49 rx->ctx.skb_head = NULL;
50 rx->ctx.skb_tail = NULL;
51
52 /* Set up linked list of buffer IDs */
53 if (rx->dqo.buf_states) {
54 for (i = 0; i < rx->dqo.num_buf_states - 1; i++)
55 rx->dqo.buf_states[i].next = i + 1;
56 rx->dqo.buf_states[rx->dqo.num_buf_states - 1].next = -1;
57 }
58
59 rx->dqo.free_buf_states = 0;
60 rx->dqo.recycled_buf_states.head = -1;
61 rx->dqo.recycled_buf_states.tail = -1;
62 rx->dqo.used_buf_states.head = -1;
63 rx->dqo.used_buf_states.tail = -1;
64 }
65
gve_rx_reset_ring_dqo(struct gve_priv * priv,int idx)66 static void gve_rx_reset_ring_dqo(struct gve_priv *priv, int idx)
67 {
68 struct gve_rx_ring *rx = &priv->rx[idx];
69 size_t size;
70 int i;
71
72 const u32 buffer_queue_slots = priv->rx_desc_cnt;
73 const u32 completion_queue_slots = priv->rx_desc_cnt;
74
75 /* Reset buffer queue */
76 if (rx->dqo.bufq.desc_ring) {
77 size = sizeof(rx->dqo.bufq.desc_ring[0]) *
78 buffer_queue_slots;
79 memset(rx->dqo.bufq.desc_ring, 0, size);
80 }
81
82 /* Reset completion queue */
83 if (rx->dqo.complq.desc_ring) {
84 size = sizeof(rx->dqo.complq.desc_ring[0]) *
85 completion_queue_slots;
86 memset(rx->dqo.complq.desc_ring, 0, size);
87 }
88
89 /* Reset q_resources */
90 if (rx->q_resources)
91 memset(rx->q_resources, 0, sizeof(*rx->q_resources));
92
93 /* Reset buf states */
94 if (rx->dqo.buf_states) {
95 for (i = 0; i < rx->dqo.num_buf_states; i++) {
96 struct gve_rx_buf_state_dqo *bs = &rx->dqo.buf_states[i];
97
98 if (rx->dqo.page_pool)
99 gve_free_to_page_pool(rx, bs, false);
100 else
101 gve_free_qpl_page_dqo(bs);
102 }
103 }
104
105 gve_rx_init_ring_state_dqo(rx, buffer_queue_slots,
106 completion_queue_slots);
107 }
108
gve_rx_stop_ring_dqo(struct gve_priv * priv,int idx)109 void gve_rx_stop_ring_dqo(struct gve_priv *priv, int idx)
110 {
111 int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx);
112 struct gve_rx_ring *rx = &priv->rx[idx];
113
114 if (!gve_rx_was_added_to_block(priv, idx))
115 return;
116
117 if (rx->dqo.page_pool)
118 page_pool_disable_direct_recycling(rx->dqo.page_pool);
119 gve_remove_napi(priv, ntfy_idx);
120 gve_rx_remove_from_block(priv, idx);
121 gve_rx_reset_ring_dqo(priv, idx);
122 }
123
gve_rx_free_ring_dqo(struct gve_priv * priv,struct gve_rx_ring * rx,struct gve_rx_alloc_rings_cfg * cfg)124 void gve_rx_free_ring_dqo(struct gve_priv *priv, struct gve_rx_ring *rx,
125 struct gve_rx_alloc_rings_cfg *cfg)
126 {
127 struct device *hdev = &priv->pdev->dev;
128 size_t completion_queue_slots;
129 size_t buffer_queue_slots;
130 int idx = rx->q_num;
131 size_t size;
132 u32 qpl_id;
133 int i;
134
135 completion_queue_slots = rx->dqo.complq.mask + 1;
136 buffer_queue_slots = rx->dqo.bufq.mask + 1;
137
138 if (rx->q_resources) {
139 dma_free_coherent(hdev, sizeof(*rx->q_resources),
140 rx->q_resources, rx->q_resources_bus);
141 rx->q_resources = NULL;
142 }
143
144 for (i = 0; i < rx->dqo.num_buf_states; i++) {
145 struct gve_rx_buf_state_dqo *bs = &rx->dqo.buf_states[i];
146
147 if (rx->dqo.page_pool)
148 gve_free_to_page_pool(rx, bs, false);
149 else
150 gve_free_qpl_page_dqo(bs);
151 }
152
153 if (rx->dqo.qpl) {
154 qpl_id = gve_get_rx_qpl_id(cfg->qcfg_tx, rx->q_num);
155 gve_free_queue_page_list(priv, rx->dqo.qpl, qpl_id);
156 rx->dqo.qpl = NULL;
157 }
158
159 if (rx->dqo.bufq.desc_ring) {
160 size = sizeof(rx->dqo.bufq.desc_ring[0]) * buffer_queue_slots;
161 dma_free_coherent(hdev, size, rx->dqo.bufq.desc_ring,
162 rx->dqo.bufq.bus);
163 rx->dqo.bufq.desc_ring = NULL;
164 }
165
166 if (rx->dqo.complq.desc_ring) {
167 size = sizeof(rx->dqo.complq.desc_ring[0]) *
168 completion_queue_slots;
169 dma_free_coherent(hdev, size, rx->dqo.complq.desc_ring,
170 rx->dqo.complq.bus);
171 rx->dqo.complq.desc_ring = NULL;
172 }
173
174 kvfree(rx->dqo.buf_states);
175 rx->dqo.buf_states = NULL;
176
177 if (rx->dqo.page_pool) {
178 page_pool_destroy(rx->dqo.page_pool);
179 rx->dqo.page_pool = NULL;
180 }
181
182 gve_rx_free_hdr_bufs(priv, rx);
183
184 netif_dbg(priv, drv, priv->dev, "freed rx ring %d\n", idx);
185 }
186
gve_rx_alloc_hdr_bufs(struct gve_priv * priv,struct gve_rx_ring * rx,const u32 buf_count)187 static int gve_rx_alloc_hdr_bufs(struct gve_priv *priv, struct gve_rx_ring *rx,
188 const u32 buf_count)
189 {
190 struct device *hdev = &priv->pdev->dev;
191
192 rx->dqo.hdr_bufs.data = dma_alloc_coherent(hdev, priv->header_buf_size * buf_count,
193 &rx->dqo.hdr_bufs.addr, GFP_KERNEL);
194 if (!rx->dqo.hdr_bufs.data)
195 return -ENOMEM;
196
197 return 0;
198 }
199
gve_rx_start_ring_dqo(struct gve_priv * priv,int idx)200 void gve_rx_start_ring_dqo(struct gve_priv *priv, int idx)
201 {
202 int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx);
203
204 gve_rx_add_to_block(priv, idx);
205 gve_add_napi(priv, ntfy_idx, gve_napi_poll_dqo);
206 }
207
gve_rx_alloc_ring_dqo(struct gve_priv * priv,struct gve_rx_alloc_rings_cfg * cfg,struct gve_rx_ring * rx,int idx)208 int gve_rx_alloc_ring_dqo(struct gve_priv *priv,
209 struct gve_rx_alloc_rings_cfg *cfg,
210 struct gve_rx_ring *rx,
211 int idx)
212 {
213 struct device *hdev = &priv->pdev->dev;
214 struct page_pool *pool;
215 int qpl_page_cnt;
216 size_t size;
217 u32 qpl_id;
218
219 const u32 buffer_queue_slots = cfg->ring_size;
220 const u32 completion_queue_slots = cfg->ring_size;
221
222 netif_dbg(priv, drv, priv->dev, "allocating rx ring DQO\n");
223
224 memset(rx, 0, sizeof(*rx));
225 rx->gve = priv;
226 rx->q_num = idx;
227
228 rx->dqo.num_buf_states = cfg->raw_addressing ? buffer_queue_slots :
229 gve_get_rx_pages_per_qpl_dqo(cfg->ring_size);
230 rx->dqo.buf_states = kvcalloc(rx->dqo.num_buf_states,
231 sizeof(rx->dqo.buf_states[0]),
232 GFP_KERNEL);
233 if (!rx->dqo.buf_states)
234 return -ENOMEM;
235
236 /* Allocate header buffers for header-split */
237 if (cfg->enable_header_split)
238 if (gve_rx_alloc_hdr_bufs(priv, rx, buffer_queue_slots))
239 goto err;
240
241 /* Allocate RX completion queue */
242 size = sizeof(rx->dqo.complq.desc_ring[0]) *
243 completion_queue_slots;
244 rx->dqo.complq.desc_ring =
245 dma_alloc_coherent(hdev, size, &rx->dqo.complq.bus, GFP_KERNEL);
246 if (!rx->dqo.complq.desc_ring)
247 goto err;
248
249 /* Allocate RX buffer queue */
250 size = sizeof(rx->dqo.bufq.desc_ring[0]) * buffer_queue_slots;
251 rx->dqo.bufq.desc_ring =
252 dma_alloc_coherent(hdev, size, &rx->dqo.bufq.bus, GFP_KERNEL);
253 if (!rx->dqo.bufq.desc_ring)
254 goto err;
255
256 if (cfg->raw_addressing) {
257 pool = gve_rx_create_page_pool(priv, rx);
258 if (IS_ERR(pool))
259 goto err;
260
261 rx->dqo.page_pool = pool;
262 } else {
263 qpl_id = gve_get_rx_qpl_id(cfg->qcfg_tx, rx->q_num);
264 qpl_page_cnt = gve_get_rx_pages_per_qpl_dqo(cfg->ring_size);
265
266 rx->dqo.qpl = gve_alloc_queue_page_list(priv, qpl_id,
267 qpl_page_cnt);
268 if (!rx->dqo.qpl)
269 goto err;
270 rx->dqo.next_qpl_page_idx = 0;
271 }
272
273 rx->q_resources = dma_alloc_coherent(hdev, sizeof(*rx->q_resources),
274 &rx->q_resources_bus, GFP_KERNEL);
275 if (!rx->q_resources)
276 goto err;
277
278 gve_rx_init_ring_state_dqo(rx, buffer_queue_slots,
279 completion_queue_slots);
280
281 return 0;
282
283 err:
284 gve_rx_free_ring_dqo(priv, rx, cfg);
285 return -ENOMEM;
286 }
287
gve_rx_write_doorbell_dqo(const struct gve_priv * priv,int queue_idx)288 void gve_rx_write_doorbell_dqo(const struct gve_priv *priv, int queue_idx)
289 {
290 const struct gve_rx_ring *rx = &priv->rx[queue_idx];
291 u64 index = be32_to_cpu(rx->q_resources->db_index);
292
293 iowrite32(rx->dqo.bufq.tail, &priv->db_bar2[index]);
294 }
295
gve_rx_alloc_rings_dqo(struct gve_priv * priv,struct gve_rx_alloc_rings_cfg * cfg)296 int gve_rx_alloc_rings_dqo(struct gve_priv *priv,
297 struct gve_rx_alloc_rings_cfg *cfg)
298 {
299 struct gve_rx_ring *rx;
300 int err;
301 int i;
302
303 rx = kvcalloc(cfg->qcfg->max_queues, sizeof(struct gve_rx_ring),
304 GFP_KERNEL);
305 if (!rx)
306 return -ENOMEM;
307
308 for (i = 0; i < cfg->qcfg->num_queues; i++) {
309 err = gve_rx_alloc_ring_dqo(priv, cfg, &rx[i], i);
310 if (err) {
311 netif_err(priv, drv, priv->dev,
312 "Failed to alloc rx ring=%d: err=%d\n",
313 i, err);
314 goto err;
315 }
316 }
317
318 cfg->rx = rx;
319 return 0;
320
321 err:
322 for (i--; i >= 0; i--)
323 gve_rx_free_ring_dqo(priv, &rx[i], cfg);
324 kvfree(rx);
325 return err;
326 }
327
gve_rx_free_rings_dqo(struct gve_priv * priv,struct gve_rx_alloc_rings_cfg * cfg)328 void gve_rx_free_rings_dqo(struct gve_priv *priv,
329 struct gve_rx_alloc_rings_cfg *cfg)
330 {
331 struct gve_rx_ring *rx = cfg->rx;
332 int i;
333
334 if (!rx)
335 return;
336
337 for (i = 0; i < cfg->qcfg->num_queues; i++)
338 gve_rx_free_ring_dqo(priv, &rx[i], cfg);
339
340 kvfree(rx);
341 cfg->rx = NULL;
342 }
343
gve_rx_post_buffers_dqo(struct gve_rx_ring * rx)344 void gve_rx_post_buffers_dqo(struct gve_rx_ring *rx)
345 {
346 struct gve_rx_compl_queue_dqo *complq = &rx->dqo.complq;
347 struct gve_rx_buf_queue_dqo *bufq = &rx->dqo.bufq;
348 struct gve_priv *priv = rx->gve;
349 u32 num_avail_slots;
350 u32 num_full_slots;
351 u32 num_posted = 0;
352
353 num_full_slots = (bufq->tail - bufq->head) & bufq->mask;
354 num_avail_slots = bufq->mask - num_full_slots;
355
356 num_avail_slots = min_t(u32, num_avail_slots, complq->num_free_slots);
357 while (num_posted < num_avail_slots) {
358 struct gve_rx_desc_dqo *desc = &bufq->desc_ring[bufq->tail];
359
360 if (unlikely(gve_alloc_buffer(rx, desc))) {
361 u64_stats_update_begin(&rx->statss);
362 rx->rx_buf_alloc_fail++;
363 u64_stats_update_end(&rx->statss);
364 break;
365 }
366
367 if (rx->dqo.hdr_bufs.data)
368 desc->header_buf_addr =
369 cpu_to_le64(rx->dqo.hdr_bufs.addr +
370 priv->header_buf_size * bufq->tail);
371
372 bufq->tail = (bufq->tail + 1) & bufq->mask;
373 complq->num_free_slots--;
374 num_posted++;
375
376 if ((bufq->tail & (GVE_RX_BUF_THRESH_DQO - 1)) == 0)
377 gve_rx_write_doorbell_dqo(priv, rx->q_num);
378 }
379
380 rx->fill_cnt += num_posted;
381 }
382
gve_rx_skb_csum(struct sk_buff * skb,const struct gve_rx_compl_desc_dqo * desc,struct gve_ptype ptype)383 static void gve_rx_skb_csum(struct sk_buff *skb,
384 const struct gve_rx_compl_desc_dqo *desc,
385 struct gve_ptype ptype)
386 {
387 skb->ip_summed = CHECKSUM_NONE;
388
389 /* HW did not identify and process L3 and L4 headers. */
390 if (unlikely(!desc->l3_l4_processed))
391 return;
392
393 if (ptype.l3_type == GVE_L3_TYPE_IPV4) {
394 if (unlikely(desc->csum_ip_err || desc->csum_external_ip_err))
395 return;
396 } else if (ptype.l3_type == GVE_L3_TYPE_IPV6) {
397 /* Checksum should be skipped if this flag is set. */
398 if (unlikely(desc->ipv6_ex_add))
399 return;
400 }
401
402 if (unlikely(desc->csum_l4_err))
403 return;
404
405 switch (ptype.l4_type) {
406 case GVE_L4_TYPE_TCP:
407 case GVE_L4_TYPE_UDP:
408 case GVE_L4_TYPE_ICMP:
409 case GVE_L4_TYPE_SCTP:
410 skb->ip_summed = CHECKSUM_UNNECESSARY;
411 break;
412 default:
413 break;
414 }
415 }
416
gve_rx_skb_hash(struct sk_buff * skb,const struct gve_rx_compl_desc_dqo * compl_desc,struct gve_ptype ptype)417 static void gve_rx_skb_hash(struct sk_buff *skb,
418 const struct gve_rx_compl_desc_dqo *compl_desc,
419 struct gve_ptype ptype)
420 {
421 enum pkt_hash_types hash_type = PKT_HASH_TYPE_L2;
422
423 if (ptype.l4_type != GVE_L4_TYPE_UNKNOWN)
424 hash_type = PKT_HASH_TYPE_L4;
425 else if (ptype.l3_type != GVE_L3_TYPE_UNKNOWN)
426 hash_type = PKT_HASH_TYPE_L3;
427
428 skb_set_hash(skb, le32_to_cpu(compl_desc->hash), hash_type);
429 }
430
gve_rx_free_skb(struct napi_struct * napi,struct gve_rx_ring * rx)431 static void gve_rx_free_skb(struct napi_struct *napi, struct gve_rx_ring *rx)
432 {
433 if (!rx->ctx.skb_head)
434 return;
435
436 if (rx->ctx.skb_head == napi->skb)
437 napi->skb = NULL;
438 dev_kfree_skb_any(rx->ctx.skb_head);
439 rx->ctx.skb_head = NULL;
440 rx->ctx.skb_tail = NULL;
441 }
442
gve_rx_should_trigger_copy_ondemand(struct gve_rx_ring * rx)443 static bool gve_rx_should_trigger_copy_ondemand(struct gve_rx_ring *rx)
444 {
445 if (!rx->dqo.qpl)
446 return false;
447 if (rx->dqo.used_buf_states_cnt <
448 (rx->dqo.num_buf_states -
449 GVE_DQO_QPL_ONDEMAND_ALLOC_THRESHOLD))
450 return false;
451 return true;
452 }
453
gve_rx_copy_ondemand(struct gve_rx_ring * rx,struct gve_rx_buf_state_dqo * buf_state,u16 buf_len)454 static int gve_rx_copy_ondemand(struct gve_rx_ring *rx,
455 struct gve_rx_buf_state_dqo *buf_state,
456 u16 buf_len)
457 {
458 struct page *page = alloc_page(GFP_ATOMIC);
459 int num_frags;
460
461 if (!page)
462 return -ENOMEM;
463
464 memcpy(page_address(page),
465 buf_state->page_info.page_address +
466 buf_state->page_info.page_offset,
467 buf_len);
468 num_frags = skb_shinfo(rx->ctx.skb_tail)->nr_frags;
469 skb_add_rx_frag(rx->ctx.skb_tail, num_frags, page,
470 0, buf_len, PAGE_SIZE);
471
472 u64_stats_update_begin(&rx->statss);
473 rx->rx_frag_alloc_cnt++;
474 u64_stats_update_end(&rx->statss);
475 /* Return unused buffer. */
476 gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states, buf_state);
477 return 0;
478 }
479
480 /* Chains multi skbs for single rx packet.
481 * Returns 0 if buffer is appended, -1 otherwise.
482 */
gve_rx_append_frags(struct napi_struct * napi,struct gve_rx_buf_state_dqo * buf_state,u16 buf_len,struct gve_rx_ring * rx,struct gve_priv * priv)483 static int gve_rx_append_frags(struct napi_struct *napi,
484 struct gve_rx_buf_state_dqo *buf_state,
485 u16 buf_len, struct gve_rx_ring *rx,
486 struct gve_priv *priv)
487 {
488 int num_frags = skb_shinfo(rx->ctx.skb_tail)->nr_frags;
489
490 if (unlikely(num_frags == MAX_SKB_FRAGS)) {
491 struct sk_buff *skb;
492
493 skb = napi_alloc_skb(napi, 0);
494 if (!skb)
495 return -1;
496
497 if (rx->dqo.page_pool)
498 skb_mark_for_recycle(skb);
499
500 if (rx->ctx.skb_tail == rx->ctx.skb_head)
501 skb_shinfo(rx->ctx.skb_head)->frag_list = skb;
502 else
503 rx->ctx.skb_tail->next = skb;
504 rx->ctx.skb_tail = skb;
505 num_frags = 0;
506 }
507 if (rx->ctx.skb_tail != rx->ctx.skb_head) {
508 rx->ctx.skb_head->len += buf_len;
509 rx->ctx.skb_head->data_len += buf_len;
510 rx->ctx.skb_head->truesize += buf_state->page_info.buf_size;
511 }
512
513 /* Trigger ondemand page allocation if we are running low on buffers */
514 if (gve_rx_should_trigger_copy_ondemand(rx))
515 return gve_rx_copy_ondemand(rx, buf_state, buf_len);
516
517 skb_add_rx_frag(rx->ctx.skb_tail, num_frags,
518 buf_state->page_info.page,
519 buf_state->page_info.page_offset,
520 buf_len, buf_state->page_info.buf_size);
521 gve_reuse_buffer(rx, buf_state);
522 return 0;
523 }
524
525 /* Returns 0 if descriptor is completed successfully.
526 * Returns -EINVAL if descriptor is invalid.
527 * Returns -ENOMEM if data cannot be copied to skb.
528 */
gve_rx_dqo(struct napi_struct * napi,struct gve_rx_ring * rx,const struct gve_rx_compl_desc_dqo * compl_desc,u32 desc_idx,int queue_idx)529 static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx,
530 const struct gve_rx_compl_desc_dqo *compl_desc,
531 u32 desc_idx, int queue_idx)
532 {
533 const u16 buffer_id = le16_to_cpu(compl_desc->buf_id);
534 const bool hbo = compl_desc->header_buffer_overflow;
535 const bool eop = compl_desc->end_of_packet != 0;
536 const bool hsplit = compl_desc->split_header;
537 struct gve_rx_buf_state_dqo *buf_state;
538 struct gve_priv *priv = rx->gve;
539 u16 buf_len;
540 u16 hdr_len;
541
542 if (unlikely(buffer_id >= rx->dqo.num_buf_states)) {
543 net_err_ratelimited("%s: Invalid RX buffer_id=%u\n",
544 priv->dev->name, buffer_id);
545 return -EINVAL;
546 }
547 buf_state = &rx->dqo.buf_states[buffer_id];
548 if (unlikely(!gve_buf_state_is_allocated(rx, buf_state))) {
549 net_err_ratelimited("%s: RX buffer_id is not allocated: %u\n",
550 priv->dev->name, buffer_id);
551 return -EINVAL;
552 }
553
554 if (unlikely(compl_desc->rx_error)) {
555 gve_free_buffer(rx, buf_state);
556 return -EINVAL;
557 }
558
559 buf_len = compl_desc->packet_len;
560 hdr_len = compl_desc->header_len;
561
562 /* Page might have not been used for awhile and was likely last written
563 * by a different thread.
564 */
565 prefetch(buf_state->page_info.page);
566
567 /* Copy the header into the skb in the case of header split */
568 if (hsplit) {
569 int unsplit = 0;
570
571 if (hdr_len && !hbo) {
572 rx->ctx.skb_head = gve_rx_copy_data(priv->dev, napi,
573 rx->dqo.hdr_bufs.data +
574 desc_idx * priv->header_buf_size,
575 hdr_len);
576 if (unlikely(!rx->ctx.skb_head))
577 goto error;
578 rx->ctx.skb_tail = rx->ctx.skb_head;
579
580 if (rx->dqo.page_pool)
581 skb_mark_for_recycle(rx->ctx.skb_head);
582 } else {
583 unsplit = 1;
584 }
585 u64_stats_update_begin(&rx->statss);
586 rx->rx_hsplit_pkt++;
587 rx->rx_hsplit_unsplit_pkt += unsplit;
588 rx->rx_hsplit_bytes += hdr_len;
589 u64_stats_update_end(&rx->statss);
590 }
591
592 /* Sync the portion of dma buffer for CPU to read. */
593 dma_sync_single_range_for_cpu(&priv->pdev->dev, buf_state->addr,
594 buf_state->page_info.page_offset,
595 buf_len, DMA_FROM_DEVICE);
596
597 /* Append to current skb if one exists. */
598 if (rx->ctx.skb_head) {
599 if (unlikely(gve_rx_append_frags(napi, buf_state, buf_len, rx,
600 priv)) != 0) {
601 goto error;
602 }
603 return 0;
604 }
605
606 if (eop && buf_len <= priv->rx_copybreak) {
607 rx->ctx.skb_head = gve_rx_copy(priv->dev, napi,
608 &buf_state->page_info, buf_len);
609 if (unlikely(!rx->ctx.skb_head))
610 goto error;
611 rx->ctx.skb_tail = rx->ctx.skb_head;
612
613 u64_stats_update_begin(&rx->statss);
614 rx->rx_copied_pkt++;
615 rx->rx_copybreak_pkt++;
616 u64_stats_update_end(&rx->statss);
617
618 gve_free_buffer(rx, buf_state);
619 return 0;
620 }
621
622 rx->ctx.skb_head = napi_get_frags(napi);
623 if (unlikely(!rx->ctx.skb_head))
624 goto error;
625 rx->ctx.skb_tail = rx->ctx.skb_head;
626
627 if (gve_rx_should_trigger_copy_ondemand(rx)) {
628 if (gve_rx_copy_ondemand(rx, buf_state, buf_len) < 0)
629 goto error;
630 return 0;
631 }
632
633 if (rx->dqo.page_pool)
634 skb_mark_for_recycle(rx->ctx.skb_head);
635
636 skb_add_rx_frag(rx->ctx.skb_head, 0, buf_state->page_info.page,
637 buf_state->page_info.page_offset, buf_len,
638 buf_state->page_info.buf_size);
639 gve_reuse_buffer(rx, buf_state);
640 return 0;
641
642 error:
643 gve_free_buffer(rx, buf_state);
644 return -ENOMEM;
645 }
646
gve_rx_complete_rsc(struct sk_buff * skb,const struct gve_rx_compl_desc_dqo * desc,struct gve_ptype ptype)647 static int gve_rx_complete_rsc(struct sk_buff *skb,
648 const struct gve_rx_compl_desc_dqo *desc,
649 struct gve_ptype ptype)
650 {
651 struct skb_shared_info *shinfo = skb_shinfo(skb);
652
653 /* Only TCP is supported right now. */
654 if (ptype.l4_type != GVE_L4_TYPE_TCP)
655 return -EINVAL;
656
657 switch (ptype.l3_type) {
658 case GVE_L3_TYPE_IPV4:
659 shinfo->gso_type = SKB_GSO_TCPV4;
660 break;
661 case GVE_L3_TYPE_IPV6:
662 shinfo->gso_type = SKB_GSO_TCPV6;
663 break;
664 default:
665 return -EINVAL;
666 }
667
668 shinfo->gso_size = le16_to_cpu(desc->rsc_seg_len);
669 return 0;
670 }
671
672 /* Returns 0 if skb is completed successfully, -1 otherwise. */
gve_rx_complete_skb(struct gve_rx_ring * rx,struct napi_struct * napi,const struct gve_rx_compl_desc_dqo * desc,netdev_features_t feat)673 static int gve_rx_complete_skb(struct gve_rx_ring *rx, struct napi_struct *napi,
674 const struct gve_rx_compl_desc_dqo *desc,
675 netdev_features_t feat)
676 {
677 struct gve_ptype ptype =
678 rx->gve->ptype_lut_dqo->ptypes[desc->packet_type];
679 int err;
680
681 skb_record_rx_queue(rx->ctx.skb_head, rx->q_num);
682
683 if (feat & NETIF_F_RXHASH)
684 gve_rx_skb_hash(rx->ctx.skb_head, desc, ptype);
685
686 if (feat & NETIF_F_RXCSUM)
687 gve_rx_skb_csum(rx->ctx.skb_head, desc, ptype);
688
689 /* RSC packets must set gso_size otherwise the TCP stack will complain
690 * that packets are larger than MTU.
691 */
692 if (desc->rsc) {
693 err = gve_rx_complete_rsc(rx->ctx.skb_head, desc, ptype);
694 if (err < 0)
695 return err;
696 }
697
698 if (skb_headlen(rx->ctx.skb_head) == 0)
699 napi_gro_frags(napi);
700 else
701 napi_gro_receive(napi, rx->ctx.skb_head);
702
703 return 0;
704 }
705
gve_rx_poll_dqo(struct gve_notify_block * block,int budget)706 int gve_rx_poll_dqo(struct gve_notify_block *block, int budget)
707 {
708 struct napi_struct *napi = &block->napi;
709 netdev_features_t feat = napi->dev->features;
710
711 struct gve_rx_ring *rx = block->rx;
712 struct gve_rx_compl_queue_dqo *complq = &rx->dqo.complq;
713
714 u32 work_done = 0;
715 u64 bytes = 0;
716 int err;
717
718 while (work_done < budget) {
719 struct gve_rx_compl_desc_dqo *compl_desc =
720 &complq->desc_ring[complq->head];
721 u32 pkt_bytes;
722
723 /* No more new packets */
724 if (compl_desc->generation == complq->cur_gen_bit)
725 break;
726
727 /* Prefetch the next two descriptors. */
728 prefetch(&complq->desc_ring[(complq->head + 1) & complq->mask]);
729 prefetch(&complq->desc_ring[(complq->head + 2) & complq->mask]);
730
731 /* Do not read data until we own the descriptor */
732 dma_rmb();
733
734 err = gve_rx_dqo(napi, rx, compl_desc, complq->head, rx->q_num);
735 if (err < 0) {
736 gve_rx_free_skb(napi, rx);
737 u64_stats_update_begin(&rx->statss);
738 if (err == -ENOMEM)
739 rx->rx_skb_alloc_fail++;
740 else if (err == -EINVAL)
741 rx->rx_desc_err_dropped_pkt++;
742 u64_stats_update_end(&rx->statss);
743 }
744
745 complq->head = (complq->head + 1) & complq->mask;
746 complq->num_free_slots++;
747
748 /* When the ring wraps, the generation bit is flipped. */
749 complq->cur_gen_bit ^= (complq->head == 0);
750
751 /* Receiving a completion means we have space to post another
752 * buffer on the buffer queue.
753 */
754 {
755 struct gve_rx_buf_queue_dqo *bufq = &rx->dqo.bufq;
756
757 bufq->head = (bufq->head + 1) & bufq->mask;
758 }
759
760 /* Free running counter of completed descriptors */
761 rx->cnt++;
762
763 if (!rx->ctx.skb_head)
764 continue;
765
766 if (!compl_desc->end_of_packet)
767 continue;
768
769 work_done++;
770 pkt_bytes = rx->ctx.skb_head->len;
771 /* The ethernet header (first ETH_HLEN bytes) is snipped off
772 * by eth_type_trans.
773 */
774 if (skb_headlen(rx->ctx.skb_head))
775 pkt_bytes += ETH_HLEN;
776
777 /* gve_rx_complete_skb() will consume skb if successful */
778 if (gve_rx_complete_skb(rx, napi, compl_desc, feat) != 0) {
779 gve_rx_free_skb(napi, rx);
780 u64_stats_update_begin(&rx->statss);
781 rx->rx_desc_err_dropped_pkt++;
782 u64_stats_update_end(&rx->statss);
783 continue;
784 }
785
786 bytes += pkt_bytes;
787 rx->ctx.skb_head = NULL;
788 rx->ctx.skb_tail = NULL;
789 }
790
791 gve_rx_post_buffers_dqo(rx);
792
793 u64_stats_update_begin(&rx->statss);
794 rx->rpackets += work_done;
795 rx->rbytes += bytes;
796 u64_stats_update_end(&rx->statss);
797
798 return work_done;
799 }
800