2 /* include/net/xdp.h
17  * DOC: XDP RX-queue information
19  * The XDP RX-queue info (xdp_rxq_info) is associated with the driver
24  * reference to this xdp_rxq_info structure.  This provides the XDP
34  * The struct is not directly tied to the XDP prog.  A new XDP prog
45 	MEM_TYPE_PAGE_ORDER0,     /* Orig XDP full page model */
51 /* XDP flags for ndo_xdp_xmit */
75 	XDP_FLAGS_HAS_FRAGS		= BIT(0), /* non-linear xdp buff */
76 	XDP_FLAGS_FRAGS_PF_MEMALLOC	= BIT(1), /* xdp paged memory is under
92 static __always_inline bool xdp_buff_has_frags(const struct xdp_buff *xdp)  in xdp_buff_has_frags()  argument
94 	return !!(xdp->flags & XDP_FLAGS_HAS_FRAGS);  in xdp_buff_has_frags()
97 static __always_inline void xdp_buff_set_frags_flag(struct xdp_buff *xdp)  in xdp_buff_set_frags_flag()  argument
99 	xdp->flags |= XDP_FLAGS_HAS_FRAGS;  in xdp_buff_set_frags_flag()
102 static __always_inline void xdp_buff_clear_frags_flag(struct xdp_buff *xdp)  in xdp_buff_clear_frags_flag()  argument
104 	xdp->flags &= ~XDP_FLAGS_HAS_FRAGS;  in xdp_buff_clear_frags_flag()
108 xdp_buff_is_frag_pfmemalloc(const struct xdp_buff *xdp)  in xdp_buff_is_frag_pfmemalloc()  argument
110 	return !!(xdp->flags & XDP_FLAGS_FRAGS_PF_MEMALLOC);  in xdp_buff_is_frag_pfmemalloc()
113 static __always_inline void xdp_buff_set_frag_pfmemalloc(struct xdp_buff *xdp)  in xdp_buff_set_frag_pfmemalloc()  argument
115 	xdp->flags |= XDP_FLAGS_FRAGS_PF_MEMALLOC;  in xdp_buff_set_frag_pfmemalloc()
119 xdp_init_buff(struct xdp_buff *xdp, u32 frame_sz, struct xdp_rxq_info *rxq)  in xdp_init_buff()  argument
121 	xdp->frame_sz = frame_sz;  in xdp_init_buff()
122 	xdp->rxq = rxq;  in xdp_init_buff()
123 	xdp->flags = 0;  in xdp_init_buff()
127 xdp_prepare_buff(struct xdp_buff *xdp, unsigned char *hard_start,  in xdp_prepare_buff()  argument
132 	xdp->data_hard_start = hard_start;  in xdp_prepare_buff()
133 	xdp->data = data;  in xdp_prepare_buff()
134 	xdp->data_end = data + data_len;  in xdp_prepare_buff()
135 	xdp->data_meta = meta_valid ? data : data + 1;  in xdp_prepare_buff()
140  * This macro reserves tailroom in the XDP buffer by limiting the
141  * XDP/BPF data access to data_hard_end.  Notice same area (and size)
144 #define xdp_data_hard_end(xdp)				\  argument
145 	((xdp)->data_hard_start + (xdp)->frame_sz -	\
149 xdp_get_shared_info_from_buff(const struct xdp_buff *xdp)  in xdp_get_shared_info_from_buff()  argument
151 	return (struct skb_shared_info *)xdp_data_hard_end(xdp);  in xdp_get_shared_info_from_buff()
155 xdp_get_buff_len(const struct xdp_buff *xdp)  in xdp_get_buff_len()  argument
157 	unsigned int len = xdp->data_end - xdp->data;  in xdp_get_buff_len()
160 	if (likely(!xdp_buff_has_frags(xdp)))  in xdp_get_buff_len()
163 	sinfo = xdp_get_shared_info_from_buff(xdp);  in xdp_get_buff_len()
169 void xdp_return_frag(netmem_ref netmem, const struct xdp_buff *xdp);
173  * @xdp: XDP buffer to attach the frag to
180  * Attach frag to the XDP buffer. If it currently has no frags attached,
190 static inline bool __xdp_buff_add_frag(struct xdp_buff *xdp, netmem_ref netmem,  in __xdp_buff_add_frag()  argument
194 	struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);  in __xdp_buff_add_frag()
198 	if (!xdp_buff_has_frags(xdp)) {  in __xdp_buff_add_frag()
199 		xdp_buff_set_frags_flag(xdp);  in __xdp_buff_add_frag()
215 		xdp_return_frag(netmem, xdp);  in __xdp_buff_add_frag()
233  * @xdp: XDP buffer to attach the frag to
244 static inline bool xdp_buff_add_frag(struct xdp_buff *xdp, netmem_ref netmem,  in xdp_buff_add_frag()  argument
247 	if (!__xdp_buff_add_frag(xdp, netmem, offset, size, truesize, true))  in xdp_buff_add_frag()
251 		xdp_buff_set_frag_pfmemalloc(xdp);  in xdp_buff_add_frag()
338 struct sk_buff *xdp_build_skb_from_buff(const struct xdp_buff *xdp);
339 struct sk_buff *xdp_build_skb_from_zc(struct xdp_buff *xdp);
340 struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp);
351 			       struct xdp_buff *xdp)  in xdp_convert_frame_to_buff()  argument
353 	xdp->data_hard_start = frame->data - frame->headroom - sizeof(*frame);  in xdp_convert_frame_to_buff()
354 	xdp->data = frame->data;  in xdp_convert_frame_to_buff()
355 	xdp->data_end = frame->data + frame->len;  in xdp_convert_frame_to_buff()
356 	xdp->data_meta = frame->data - frame->metasize;  in xdp_convert_frame_to_buff()
357 	xdp->frame_sz = frame->frame_sz;  in xdp_convert_frame_to_buff()
358 	xdp->flags = frame->flags;  in xdp_convert_frame_to_buff()
362 int xdp_update_frame_from_buff(const struct xdp_buff *xdp,  in xdp_update_frame_from_buff()  argument
368 	headroom = xdp->data - xdp->data_hard_start;  in xdp_update_frame_from_buff()
369 	metasize = xdp->data - xdp->data_meta;  in xdp_update_frame_from_buff()
375 	if (unlikely(xdp->data_end > xdp_data_hard_end(xdp))) {  in xdp_update_frame_from_buff()
380 	xdp_frame->data = xdp->data;  in xdp_update_frame_from_buff()
381 	xdp_frame->len  = xdp->data_end - xdp->data;  in xdp_update_frame_from_buff()
384 	xdp_frame->frame_sz = xdp->frame_sz;  in xdp_update_frame_from_buff()
385 	xdp_frame->flags = xdp->flags;  in xdp_update_frame_from_buff()
392 struct xdp_frame *xdp_convert_buff_to_frame(struct xdp_buff *xdp)  in xdp_convert_buff_to_frame()  argument
396 	if (xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL)  in xdp_convert_buff_to_frame()
397 		return xdp_convert_zc_to_xdp_frame(xdp);  in xdp_convert_buff_to_frame()
400 	xdp_frame = xdp->data_hard_start;  in xdp_convert_buff_to_frame()
401 	if (unlikely(xdp_update_frame_from_buff(xdp, xdp_frame) < 0))  in xdp_convert_buff_to_frame()
405 	xdp_frame->mem_type = xdp->rxq->mem.type;  in xdp_convert_buff_to_frame()
411 		  bool napi_direct, struct xdp_buff *xdp);
414 void xdp_return_buff(struct xdp_buff *xdp);
469  * @xdp_rxq: XDP RxQ info to attach the memory info to
484  * @xdp_rxq: XDP RxQ info to detach the memory info from
495 /* Drivers not supporting XDP metadata can use this helper, which
499 xdp_set_data_meta_invalid(struct xdp_buff *xdp)  in xdp_set_data_meta_invalid()  argument
501 	xdp->data_meta = xdp->data + 1;  in xdp_set_data_meta_invalid()
505 xdp_data_meta_unsupported(const struct xdp_buff *xdp)  in xdp_data_meta_unsupported()  argument
507 	return unlikely(xdp->data_meta > xdp->data);  in xdp_data_meta_unsupported()
531 /* Define the relationship between xdp-rx-metadata kfunc and
648 					    struct xdp_buff *xdp)  in bpf_prog_run_xdp()  argument
650 	/* Driver XDP hooks are invoked within a single NAPI poll cycle and thus  in bpf_prog_run_xdp()
654 	u32 act = __bpf_prog_run(prog, xdp, BPF_DISPATCHER_FUNC(xdp));  in bpf_prog_run_xdp()
657 		if (act == XDP_TX && netif_is_bond_slave(xdp->rxq->dev))  in bpf_prog_run_xdp()
658 			act = xdp_master_redirect(xdp);  in bpf_prog_run_xdp()