2 /* net/core/xdp.c
20 #include <net/xdp.h>
22 #include <trace/events/xdp.h>
371  * xdp_reg_page_pool - register &page_pool as a memory provider for XDP
374  * Can be used to register pools manually without connecting to any XDP RxQ
375  * info, so that the XDP layer will be aware of them. Then, they can be
408  * @xdp_rxq: XDP RxQ info to attach the pool to
426 /* XDP RX runs under NAPI protection, and in different delivery error
433 		  bool napi_direct, struct xdp_buff *xdp)  in __xdp_return()  argument
454 		xsk_buff_free(xdp);  in __xdp_return()
458 		WARN(1, "Incorrect XDP memory type (%d) usage", mem_type);  in __xdp_return()
497 /* XDP bulk APIs introduce a defer/flush mechanism to return
538  * xdp_return_frag -- free one XDP frag or decrement its refcount
540  * @xdp: &xdp_buff to release the frag for
542 void xdp_return_frag(netmem_ref netmem, const struct xdp_buff *xdp)  in xdp_return_frag()  argument
544 	__xdp_return(netmem, xdp->rxq->mem.type, true, NULL);  in xdp_return_frag()
548 void xdp_return_buff(struct xdp_buff *xdp)  in xdp_return_buff()  argument
552 	if (likely(!xdp_buff_has_frags(xdp)))  in xdp_return_buff()
555 	sinfo = xdp_get_shared_info_from_buff(xdp);  in xdp_return_buff()
558 			     xdp->rxq->mem.type, true, xdp);  in xdp_return_buff()
561 	__xdp_return(virt_to_netmem(xdp->data), xdp->rxq->mem.type, true, xdp);  in xdp_return_buff()
575 struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp)  in xdp_convert_zc_to_xdp_frame()  argument
583 	metasize = xdp_data_meta_unsupported(xdp) ? 0 :  in xdp_convert_zc_to_xdp_frame()
584 		   xdp->data - xdp->data_meta;  in xdp_convert_zc_to_xdp_frame()
585 	totsize = xdp->data_end - xdp->data + metasize;  in xdp_convert_zc_to_xdp_frame()
599 	data_to_copy = metasize ? xdp->data_meta : xdp->data;  in xdp_convert_zc_to_xdp_frame()
609 	xsk_buff_free(xdp);  in xdp_convert_zc_to_xdp_frame()
633  * @xdp: &xdp_buff to convert to an skb
642 struct sk_buff *xdp_build_skb_from_buff(const struct xdp_buff *xdp)  in xdp_build_skb_from_buff()  argument
644 	const struct xdp_rxq_info *rxq = xdp->rxq;  in xdp_build_skb_from_buff()
650 	if (unlikely(xdp_buff_has_frags(xdp))) {  in xdp_build_skb_from_buff()
651 		sinfo = xdp_get_shared_info_from_buff(xdp);  in xdp_build_skb_from_buff()
655 	skb = napi_build_skb(xdp->data_hard_start, xdp->frame_sz);  in xdp_build_skb_from_buff()
659 	skb_reserve(skb, xdp->data - xdp->data_hard_start);  in xdp_build_skb_from_buff()
660 	__skb_put(skb, xdp->data_end - xdp->data);  in xdp_build_skb_from_buff()
662 	metalen = xdp->data - xdp->data_meta;  in xdp_build_skb_from_buff()
674 		tsize = sinfo->xdp_frags_truesize ? : nr_frags * xdp->frame_sz;  in xdp_build_skb_from_buff()
677 					   xdp_buff_is_frag_pfmemalloc(xdp));  in xdp_build_skb_from_buff()
689  * @xdp: XSk &xdp_buff from which the frags will be copied
698 					    const struct xdp_buff *xdp,  in xdp_copy_frags_from_zc()  argument
706 	xinfo = xdp_get_shared_info_from_buff(xdp);  in xdp_copy_frags_from_zc()
737  * @xdp: source XSk buff
743  * If new skb was built successfully, @xdp is returned to XSk pool's freelist.
748 struct sk_buff *xdp_build_skb_from_zc(struct xdp_buff *xdp)  in xdp_build_skb_from_zc()  argument
751 	const struct xdp_rxq_info *rxq = xdp->rxq;  in xdp_build_skb_from_zc()
752 	u32 len = xdp->data_end - xdp->data_meta;  in xdp_build_skb_from_zc()
753 	u32 truesize = xdp->frame_sz;  in xdp_build_skb_from_zc()
772 	skb_reserve(skb, xdp->data_meta - xdp->data_hard_start);  in xdp_build_skb_from_zc()
774 	memcpy(__skb_put(skb, len), xdp->data_meta, LARGEST_ALIGN(len));  in xdp_build_skb_from_zc()
776 	metalen = xdp->data - xdp->data_meta;  in xdp_build_skb_from_zc()
784 	if (unlikely(xdp_buff_has_frags(xdp)) &&  in xdp_build_skb_from_zc()
785 	    unlikely(!xdp_copy_frags_from_zc(skb, xdp, pp))) {  in xdp_build_skb_from_zc()
790 	xsk_buff_free(xdp);  in xdp_build_skb_from_zc()
807 	/* xdp frags frame */  in __xdp_build_skb_from_frame()
899  * bpf_xdp_metadata_rx_timestamp - Read XDP frame RX timestamp.
900  * @ctx: XDP context pointer.
914  * bpf_xdp_metadata_rx_hash - Read XDP frame RX hash.
915  * @ctx: XDP context pointer.
937  * bpf_xdp_metadata_rx_vlan_tag - Get XDP packet outermost VLAN tag
938  * @ctx: XDP context pointer.