Lines Matching +full:xdp +full:- +full:rx +full:- +full:metadata
1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /* include/net/xdp.h
17 * DOC: XDP RX-queue information
19 * The XDP RX-queue info (xdp_rxq_info) is associated with the driver
20 * level RX-ring queues. It is information that is specific to how
21 * the driver has configured a given RX-ring queue.
24 * reference to this xdp_rxq_info structure. This provides the XDP
25 * data-path read-access to RX-info for both kernel and bpf-side
29 * context. Contents are read-mostly and must not be updated during
34 * The struct is not directly tied to the XDP prog. A new XDP prog
36 * RX-ring. If the RX-ring does change significantly, the NIC driver
37 * naturally needs to stop the RX-ring before purging and reallocating
40 * also mandatory during RX-ring setup.
44 MEM_TYPE_PAGE_SHARED = 0, /* Split-page refcnt based model */
45 MEM_TYPE_PAGE_ORDER0, /* Orig XDP full page model */
51 /* XDP flags for ndo_xdp_xmit */
68 } ____cacheline_aligned; /* perf critical, avoid false-sharing */
75 XDP_FLAGS_HAS_FRAGS = BIT(0), /* non-linear xdp buff */
76 XDP_FLAGS_FRAGS_PF_MEMALLOC = BIT(1), /* xdp paged memory is under
92 static __always_inline bool xdp_buff_has_frags(const struct xdp_buff *xdp) in xdp_buff_has_frags() argument
94 return !!(xdp->flags & XDP_FLAGS_HAS_FRAGS); in xdp_buff_has_frags()
97 static __always_inline void xdp_buff_set_frags_flag(struct xdp_buff *xdp) in xdp_buff_set_frags_flag() argument
99 xdp->flags |= XDP_FLAGS_HAS_FRAGS; in xdp_buff_set_frags_flag()
102 static __always_inline void xdp_buff_clear_frags_flag(struct xdp_buff *xdp) in xdp_buff_clear_frags_flag() argument
104 xdp->flags &= ~XDP_FLAGS_HAS_FRAGS; in xdp_buff_clear_frags_flag()
108 xdp_buff_is_frag_pfmemalloc(const struct xdp_buff *xdp) in xdp_buff_is_frag_pfmemalloc() argument
110 return !!(xdp->flags & XDP_FLAGS_FRAGS_PF_MEMALLOC); in xdp_buff_is_frag_pfmemalloc()
113 static __always_inline void xdp_buff_set_frag_pfmemalloc(struct xdp_buff *xdp) in xdp_buff_set_frag_pfmemalloc() argument
115 xdp->flags |= XDP_FLAGS_FRAGS_PF_MEMALLOC; in xdp_buff_set_frag_pfmemalloc()
119 xdp_init_buff(struct xdp_buff *xdp, u32 frame_sz, struct xdp_rxq_info *rxq) in xdp_init_buff() argument
121 xdp->frame_sz = frame_sz; in xdp_init_buff()
122 xdp->rxq = rxq; in xdp_init_buff()
123 xdp->flags = 0; in xdp_init_buff()
127 xdp_prepare_buff(struct xdp_buff *xdp, unsigned char *hard_start, in xdp_prepare_buff() argument
132 xdp->data_hard_start = hard_start; in xdp_prepare_buff()
133 xdp->data = data; in xdp_prepare_buff()
134 xdp->data_end = data + data_len; in xdp_prepare_buff()
135 xdp->data_meta = meta_valid ? data : data + 1; in xdp_prepare_buff()
138 /* Reserve memory area at end-of data area.
140 * This macro reserves tailroom in the XDP buffer by limiting the
141 * XDP/BPF data access to data_hard_end. Notice same area (and size)
144 #define xdp_data_hard_end(xdp) \ argument
145 ((xdp)->data_hard_start + (xdp)->frame_sz - \
149 xdp_get_shared_info_from_buff(const struct xdp_buff *xdp) in xdp_get_shared_info_from_buff() argument
151 return (struct skb_shared_info *)xdp_data_hard_end(xdp); in xdp_get_shared_info_from_buff()
155 xdp_get_buff_len(const struct xdp_buff *xdp) in xdp_get_buff_len() argument
157 unsigned int len = xdp->data_end - xdp->data; in xdp_get_buff_len()
160 if (likely(!xdp_buff_has_frags(xdp))) in xdp_get_buff_len()
163 sinfo = xdp_get_shared_info_from_buff(xdp); in xdp_get_buff_len()
164 len += sinfo->xdp_frags_size; in xdp_get_buff_len()
169 void xdp_return_frag(netmem_ref netmem, const struct xdp_buff *xdp);
172 * __xdp_buff_add_frag - attach frag to &xdp_buff
173 * @xdp: XDP buffer to attach the frag to
180 * Attach frag to the XDP buffer. If it currently has no frags attached,
185 * non-underscored wrapper in drivers.
190 static inline bool __xdp_buff_add_frag(struct xdp_buff *xdp, netmem_ref netmem, in __xdp_buff_add_frag() argument
194 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); in __xdp_buff_add_frag()
198 if (!xdp_buff_has_frags(xdp)) { in __xdp_buff_add_frag()
199 xdp_buff_set_frags_flag(xdp); in __xdp_buff_add_frag()
202 sinfo->xdp_frags_size = 0; in __xdp_buff_add_frag()
203 sinfo->xdp_frags_truesize = 0; in __xdp_buff_add_frag()
208 nr_frags = sinfo->nr_frags; in __xdp_buff_add_frag()
209 prev = &sinfo->frags[nr_frags - 1]; in __xdp_buff_add_frag()
215 xdp_return_frag(netmem, xdp); in __xdp_buff_add_frag()
224 sinfo->nr_frags = nr_frags; in __xdp_buff_add_frag()
225 sinfo->xdp_frags_size += size; in __xdp_buff_add_frag()
226 sinfo->xdp_frags_truesize += truesize; in __xdp_buff_add_frag()
232 * xdp_buff_add_frag - attach frag to &xdp_buff
233 * @xdp: XDP buffer to attach the frag to
244 static inline bool xdp_buff_add_frag(struct xdp_buff *xdp, netmem_ref netmem, in xdp_buff_add_frag() argument
247 if (!__xdp_buff_add_frag(xdp, netmem, offset, size, truesize, true)) in xdp_buff_add_frag()
251 xdp_buff_set_frag_pfmemalloc(xdp); in xdp_buff_add_frag()
260 u32 metasize; /* uses lower 8-bits */
272 return !!(frame->flags & XDP_FLAGS_HAS_FRAGS); in xdp_frame_has_frags()
278 return !!(frame->flags & XDP_FLAGS_FRAGS_PF_MEMALLOC); in xdp_frame_is_frag_pfmemalloc()
289 bq->count = 0; in xdp_frame_bulk_init()
295 void *data_hard_start = frame->data - frame->headroom - sizeof(*frame); in xdp_get_shared_info_from_frame()
297 return (struct skb_shared_info *)(data_hard_start + frame->frame_sz - in xdp_get_shared_info_from_frame()
310 frame->data = NULL; in xdp_scrub_frame()
311 frame->dev_rx = NULL; in xdp_scrub_frame()
321 sinfo->nr_frags = nr_frags; in xdp_update_skb_shared_info()
326 sinfo->destructor_arg = NULL; in xdp_update_skb_shared_info()
328 skb->len += size; in xdp_update_skb_shared_info()
329 skb->data_len += size; in xdp_update_skb_shared_info()
330 skb->truesize += truesize; in xdp_update_skb_shared_info()
331 skb->pfmemalloc |= pfmemalloc; in xdp_update_skb_shared_info()
334 /* Avoids inlining WARN macro in fast-path */
338 struct sk_buff *xdp_build_skb_from_buff(const struct xdp_buff *xdp);
339 struct sk_buff *xdp_build_skb_from_zc(struct xdp_buff *xdp);
340 struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp);
351 struct xdp_buff *xdp) in xdp_convert_frame_to_buff() argument
353 xdp->data_hard_start = frame->data - frame->headroom - sizeof(*frame); in xdp_convert_frame_to_buff()
354 xdp->data = frame->data; in xdp_convert_frame_to_buff()
355 xdp->data_end = frame->data + frame->len; in xdp_convert_frame_to_buff()
356 xdp->data_meta = frame->data - frame->metasize; in xdp_convert_frame_to_buff()
357 xdp->frame_sz = frame->frame_sz; in xdp_convert_frame_to_buff()
358 xdp->flags = frame->flags; in xdp_convert_frame_to_buff()
362 int xdp_update_frame_from_buff(const struct xdp_buff *xdp, in xdp_update_frame_from_buff() argument
368 headroom = xdp->data - xdp->data_hard_start; in xdp_update_frame_from_buff()
369 metasize = xdp->data - xdp->data_meta; in xdp_update_frame_from_buff()
371 if (unlikely((headroom - metasize) < sizeof(*xdp_frame))) in xdp_update_frame_from_buff()
372 return -ENOSPC; in xdp_update_frame_from_buff()
375 if (unlikely(xdp->data_end > xdp_data_hard_end(xdp))) { in xdp_update_frame_from_buff()
377 return -ENOSPC; in xdp_update_frame_from_buff()
380 xdp_frame->data = xdp->data; in xdp_update_frame_from_buff()
381 xdp_frame->len = xdp->data_end - xdp->data; in xdp_update_frame_from_buff()
382 xdp_frame->headroom = headroom - sizeof(*xdp_frame); in xdp_update_frame_from_buff()
383 xdp_frame->metasize = metasize; in xdp_update_frame_from_buff()
384 xdp_frame->frame_sz = xdp->frame_sz; in xdp_update_frame_from_buff()
385 xdp_frame->flags = xdp->flags; in xdp_update_frame_from_buff()
392 struct xdp_frame *xdp_convert_buff_to_frame(struct xdp_buff *xdp) in xdp_convert_buff_to_frame() argument
396 if (xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL) in xdp_convert_buff_to_frame()
397 return xdp_convert_zc_to_xdp_frame(xdp); in xdp_convert_buff_to_frame()
400 xdp_frame = xdp->data_hard_start; in xdp_convert_buff_to_frame()
401 if (unlikely(xdp_update_frame_from_buff(xdp, xdp_frame) < 0)) in xdp_convert_buff_to_frame()
405 xdp_frame->mem_type = xdp->rxq->mem.type; in xdp_convert_buff_to_frame()
411 bool napi_direct, struct xdp_buff *xdp);
414 void xdp_return_buff(struct xdp_buff *xdp);
420 if (unlikely(!bq->count)) in xdp_flush_frame_bulk()
423 page_pool_put_netmem_bulk(bq->q, bq->count); in xdp_flush_frame_bulk()
424 bq->count = 0; in xdp_flush_frame_bulk()
431 unsigned int len = xdpf->len; in xdp_get_frame_len()
437 len += sinfo->xdp_frags_size; in xdp_get_frame_len()
468 * xdp_rxq_info_attach_mem_model - attach registered mem info to RxQ info
469 * @xdp_rxq: XDP RxQ info to attach the memory info to
479 xdp_rxq->mem = *mem; in xdp_rxq_info_attach_mem_model()
483 * xdp_rxq_info_detach_mem_model - detach registered mem info from RxQ info
484 * @xdp_rxq: XDP RxQ info to detach the memory info from
492 xdp_rxq->mem = (struct xdp_mem_info){ }; in xdp_rxq_info_detach_mem_model()
495 /* Drivers not supporting XDP metadata can use this helper, which
496 * rejects any room expansion for metadata as a result.
499 xdp_set_data_meta_invalid(struct xdp_buff *xdp) in xdp_set_data_meta_invalid() argument
501 xdp->data_meta = xdp->data + 1; in xdp_set_data_meta_invalid()
505 xdp_data_meta_unsupported(const struct xdp_buff *xdp) in xdp_data_meta_unsupported() argument
507 return unlikely(xdp->data_meta > xdp->data); in xdp_data_meta_unsupported()
531 /* Define the relationship between xdp-rx-metadata kfunc and
533 * - xdp_rx_metadata enum
534 * - netdev netlink enum (Documentation/netlink/specs/netdev.yaml)
535 * - kfunc name
536 * - xdp_metadata_ops field
648 struct xdp_buff *xdp) in bpf_prog_run_xdp() argument
650 /* Driver XDP hooks are invoked within a single NAPI poll cycle and thus in bpf_prog_run_xdp()
654 u32 act = __bpf_prog_run(prog, xdp, BPF_DISPATCHER_FUNC(xdp)); in bpf_prog_run_xdp()
657 if (act == XDP_TX && netif_is_bond_slave(xdp->rxq->dev)) in bpf_prog_run_xdp()
658 act = xdp_master_redirect(xdp); in bpf_prog_run_xdp()