xref: /aosp_15_r20/external/linux-kselftest/tools/testing/selftests/bpf/xsk.c (revision 053f45be4e351dfd5e965df293cd45b779f579ee)
1*053f45beSAndroid Build Coastguard Worker // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2*053f45beSAndroid Build Coastguard Worker 
3*053f45beSAndroid Build Coastguard Worker /*
4*053f45beSAndroid Build Coastguard Worker  * AF_XDP user-space access library.
5*053f45beSAndroid Build Coastguard Worker  *
6*053f45beSAndroid Build Coastguard Worker  * Copyright(c) 2018 - 2019 Intel Corporation.
7*053f45beSAndroid Build Coastguard Worker  *
8*053f45beSAndroid Build Coastguard Worker  * Author(s): Magnus Karlsson <[email protected]>
9*053f45beSAndroid Build Coastguard Worker  */
10*053f45beSAndroid Build Coastguard Worker 
11*053f45beSAndroid Build Coastguard Worker #include <errno.h>
12*053f45beSAndroid Build Coastguard Worker #include <stdlib.h>
13*053f45beSAndroid Build Coastguard Worker #include <string.h>
14*053f45beSAndroid Build Coastguard Worker #include <unistd.h>
15*053f45beSAndroid Build Coastguard Worker #include <arpa/inet.h>
16*053f45beSAndroid Build Coastguard Worker #include <asm/barrier.h>
17*053f45beSAndroid Build Coastguard Worker #include <linux/compiler.h>
18*053f45beSAndroid Build Coastguard Worker #include <linux/ethtool.h>
19*053f45beSAndroid Build Coastguard Worker #include <linux/filter.h>
20*053f45beSAndroid Build Coastguard Worker #include <linux/if_ether.h>
21*053f45beSAndroid Build Coastguard Worker #include <linux/if_packet.h>
22*053f45beSAndroid Build Coastguard Worker #include <linux/if_xdp.h>
23*053f45beSAndroid Build Coastguard Worker #include <linux/kernel.h>
24*053f45beSAndroid Build Coastguard Worker #include <linux/list.h>
25*053f45beSAndroid Build Coastguard Worker #include <linux/sockios.h>
26*053f45beSAndroid Build Coastguard Worker #include <net/if.h>
27*053f45beSAndroid Build Coastguard Worker #include <sys/ioctl.h>
28*053f45beSAndroid Build Coastguard Worker #include <sys/mman.h>
29*053f45beSAndroid Build Coastguard Worker #include <sys/socket.h>
30*053f45beSAndroid Build Coastguard Worker #include <sys/types.h>
31*053f45beSAndroid Build Coastguard Worker #include <linux/if_link.h>
32*053f45beSAndroid Build Coastguard Worker 
33*053f45beSAndroid Build Coastguard Worker #include <bpf/bpf.h>
34*053f45beSAndroid Build Coastguard Worker #include <bpf/libbpf.h>
35*053f45beSAndroid Build Coastguard Worker #include "xsk.h"
36*053f45beSAndroid Build Coastguard Worker 
37*053f45beSAndroid Build Coastguard Worker #ifndef SOL_XDP
38*053f45beSAndroid Build Coastguard Worker  #define SOL_XDP 283
39*053f45beSAndroid Build Coastguard Worker #endif
40*053f45beSAndroid Build Coastguard Worker 
41*053f45beSAndroid Build Coastguard Worker #ifndef AF_XDP
42*053f45beSAndroid Build Coastguard Worker  #define AF_XDP 44
43*053f45beSAndroid Build Coastguard Worker #endif
44*053f45beSAndroid Build Coastguard Worker 
45*053f45beSAndroid Build Coastguard Worker #ifndef PF_XDP
46*053f45beSAndroid Build Coastguard Worker  #define PF_XDP AF_XDP
47*053f45beSAndroid Build Coastguard Worker #endif
48*053f45beSAndroid Build Coastguard Worker 
49*053f45beSAndroid Build Coastguard Worker #define pr_warn(fmt, ...) fprintf(stderr, fmt, ##__VA_ARGS__)
50*053f45beSAndroid Build Coastguard Worker 
51*053f45beSAndroid Build Coastguard Worker enum xsk_prog {
52*053f45beSAndroid Build Coastguard Worker 	XSK_PROG_FALLBACK,
53*053f45beSAndroid Build Coastguard Worker 	XSK_PROG_REDIRECT_FLAGS,
54*053f45beSAndroid Build Coastguard Worker };
55*053f45beSAndroid Build Coastguard Worker 
56*053f45beSAndroid Build Coastguard Worker struct xsk_umem {
57*053f45beSAndroid Build Coastguard Worker 	struct xsk_ring_prod *fill_save;
58*053f45beSAndroid Build Coastguard Worker 	struct xsk_ring_cons *comp_save;
59*053f45beSAndroid Build Coastguard Worker 	char *umem_area;
60*053f45beSAndroid Build Coastguard Worker 	struct xsk_umem_config config;
61*053f45beSAndroid Build Coastguard Worker 	int fd;
62*053f45beSAndroid Build Coastguard Worker 	int refcount;
63*053f45beSAndroid Build Coastguard Worker 	struct list_head ctx_list;
64*053f45beSAndroid Build Coastguard Worker 	bool rx_ring_setup_done;
65*053f45beSAndroid Build Coastguard Worker 	bool tx_ring_setup_done;
66*053f45beSAndroid Build Coastguard Worker };
67*053f45beSAndroid Build Coastguard Worker 
68*053f45beSAndroid Build Coastguard Worker struct xsk_ctx {
69*053f45beSAndroid Build Coastguard Worker 	struct xsk_ring_prod *fill;
70*053f45beSAndroid Build Coastguard Worker 	struct xsk_ring_cons *comp;
71*053f45beSAndroid Build Coastguard Worker 	__u32 queue_id;
72*053f45beSAndroid Build Coastguard Worker 	struct xsk_umem *umem;
73*053f45beSAndroid Build Coastguard Worker 	int refcount;
74*053f45beSAndroid Build Coastguard Worker 	int ifindex;
75*053f45beSAndroid Build Coastguard Worker 	struct list_head list;
76*053f45beSAndroid Build Coastguard Worker 	int prog_fd;
77*053f45beSAndroid Build Coastguard Worker 	int link_fd;
78*053f45beSAndroid Build Coastguard Worker 	int xsks_map_fd;
79*053f45beSAndroid Build Coastguard Worker 	char ifname[IFNAMSIZ];
80*053f45beSAndroid Build Coastguard Worker 	bool has_bpf_link;
81*053f45beSAndroid Build Coastguard Worker };
82*053f45beSAndroid Build Coastguard Worker 
83*053f45beSAndroid Build Coastguard Worker struct xsk_socket {
84*053f45beSAndroid Build Coastguard Worker 	struct xsk_ring_cons *rx;
85*053f45beSAndroid Build Coastguard Worker 	struct xsk_ring_prod *tx;
86*053f45beSAndroid Build Coastguard Worker 	__u64 outstanding_tx;
87*053f45beSAndroid Build Coastguard Worker 	struct xsk_ctx *ctx;
88*053f45beSAndroid Build Coastguard Worker 	struct xsk_socket_config config;
89*053f45beSAndroid Build Coastguard Worker 	int fd;
90*053f45beSAndroid Build Coastguard Worker };
91*053f45beSAndroid Build Coastguard Worker 
92*053f45beSAndroid Build Coastguard Worker struct xsk_nl_info {
93*053f45beSAndroid Build Coastguard Worker 	bool xdp_prog_attached;
94*053f45beSAndroid Build Coastguard Worker 	int ifindex;
95*053f45beSAndroid Build Coastguard Worker 	int fd;
96*053f45beSAndroid Build Coastguard Worker };
97*053f45beSAndroid Build Coastguard Worker 
98*053f45beSAndroid Build Coastguard Worker /* Up until and including Linux 5.3 */
99*053f45beSAndroid Build Coastguard Worker struct xdp_ring_offset_v1 {
100*053f45beSAndroid Build Coastguard Worker 	__u64 producer;
101*053f45beSAndroid Build Coastguard Worker 	__u64 consumer;
102*053f45beSAndroid Build Coastguard Worker 	__u64 desc;
103*053f45beSAndroid Build Coastguard Worker };
104*053f45beSAndroid Build Coastguard Worker 
105*053f45beSAndroid Build Coastguard Worker /* Up until and including Linux 5.3 */
106*053f45beSAndroid Build Coastguard Worker struct xdp_mmap_offsets_v1 {
107*053f45beSAndroid Build Coastguard Worker 	struct xdp_ring_offset_v1 rx;
108*053f45beSAndroid Build Coastguard Worker 	struct xdp_ring_offset_v1 tx;
109*053f45beSAndroid Build Coastguard Worker 	struct xdp_ring_offset_v1 fr;
110*053f45beSAndroid Build Coastguard Worker 	struct xdp_ring_offset_v1 cr;
111*053f45beSAndroid Build Coastguard Worker };
112*053f45beSAndroid Build Coastguard Worker 
xsk_umem__fd(const struct xsk_umem * umem)113*053f45beSAndroid Build Coastguard Worker int xsk_umem__fd(const struct xsk_umem *umem)
114*053f45beSAndroid Build Coastguard Worker {
115*053f45beSAndroid Build Coastguard Worker 	return umem ? umem->fd : -EINVAL;
116*053f45beSAndroid Build Coastguard Worker }
117*053f45beSAndroid Build Coastguard Worker 
xsk_socket__fd(const struct xsk_socket * xsk)118*053f45beSAndroid Build Coastguard Worker int xsk_socket__fd(const struct xsk_socket *xsk)
119*053f45beSAndroid Build Coastguard Worker {
120*053f45beSAndroid Build Coastguard Worker 	return xsk ? xsk->fd : -EINVAL;
121*053f45beSAndroid Build Coastguard Worker }
122*053f45beSAndroid Build Coastguard Worker 
xsk_page_aligned(void * buffer)123*053f45beSAndroid Build Coastguard Worker static bool xsk_page_aligned(void *buffer)
124*053f45beSAndroid Build Coastguard Worker {
125*053f45beSAndroid Build Coastguard Worker 	unsigned long addr = (unsigned long)buffer;
126*053f45beSAndroid Build Coastguard Worker 
127*053f45beSAndroid Build Coastguard Worker 	return !(addr & (getpagesize() - 1));
128*053f45beSAndroid Build Coastguard Worker }
129*053f45beSAndroid Build Coastguard Worker 
xsk_set_umem_config(struct xsk_umem_config * cfg,const struct xsk_umem_config * usr_cfg)130*053f45beSAndroid Build Coastguard Worker static void xsk_set_umem_config(struct xsk_umem_config *cfg,
131*053f45beSAndroid Build Coastguard Worker 				const struct xsk_umem_config *usr_cfg)
132*053f45beSAndroid Build Coastguard Worker {
133*053f45beSAndroid Build Coastguard Worker 	if (!usr_cfg) {
134*053f45beSAndroid Build Coastguard Worker 		cfg->fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
135*053f45beSAndroid Build Coastguard Worker 		cfg->comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS;
136*053f45beSAndroid Build Coastguard Worker 		cfg->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
137*053f45beSAndroid Build Coastguard Worker 		cfg->frame_headroom = XSK_UMEM__DEFAULT_FRAME_HEADROOM;
138*053f45beSAndroid Build Coastguard Worker 		cfg->flags = XSK_UMEM__DEFAULT_FLAGS;
139*053f45beSAndroid Build Coastguard Worker 		return;
140*053f45beSAndroid Build Coastguard Worker 	}
141*053f45beSAndroid Build Coastguard Worker 
142*053f45beSAndroid Build Coastguard Worker 	cfg->fill_size = usr_cfg->fill_size;
143*053f45beSAndroid Build Coastguard Worker 	cfg->comp_size = usr_cfg->comp_size;
144*053f45beSAndroid Build Coastguard Worker 	cfg->frame_size = usr_cfg->frame_size;
145*053f45beSAndroid Build Coastguard Worker 	cfg->frame_headroom = usr_cfg->frame_headroom;
146*053f45beSAndroid Build Coastguard Worker 	cfg->flags = usr_cfg->flags;
147*053f45beSAndroid Build Coastguard Worker }
148*053f45beSAndroid Build Coastguard Worker 
xsk_set_xdp_socket_config(struct xsk_socket_config * cfg,const struct xsk_socket_config * usr_cfg)149*053f45beSAndroid Build Coastguard Worker static int xsk_set_xdp_socket_config(struct xsk_socket_config *cfg,
150*053f45beSAndroid Build Coastguard Worker 				     const struct xsk_socket_config *usr_cfg)
151*053f45beSAndroid Build Coastguard Worker {
152*053f45beSAndroid Build Coastguard Worker 	if (!usr_cfg) {
153*053f45beSAndroid Build Coastguard Worker 		cfg->rx_size = XSK_RING_CONS__DEFAULT_NUM_DESCS;
154*053f45beSAndroid Build Coastguard Worker 		cfg->tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
155*053f45beSAndroid Build Coastguard Worker 		cfg->libbpf_flags = 0;
156*053f45beSAndroid Build Coastguard Worker 		cfg->xdp_flags = 0;
157*053f45beSAndroid Build Coastguard Worker 		cfg->bind_flags = 0;
158*053f45beSAndroid Build Coastguard Worker 		return 0;
159*053f45beSAndroid Build Coastguard Worker 	}
160*053f45beSAndroid Build Coastguard Worker 
161*053f45beSAndroid Build Coastguard Worker 	if (usr_cfg->libbpf_flags & ~XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD)
162*053f45beSAndroid Build Coastguard Worker 		return -EINVAL;
163*053f45beSAndroid Build Coastguard Worker 
164*053f45beSAndroid Build Coastguard Worker 	cfg->rx_size = usr_cfg->rx_size;
165*053f45beSAndroid Build Coastguard Worker 	cfg->tx_size = usr_cfg->tx_size;
166*053f45beSAndroid Build Coastguard Worker 	cfg->libbpf_flags = usr_cfg->libbpf_flags;
167*053f45beSAndroid Build Coastguard Worker 	cfg->xdp_flags = usr_cfg->xdp_flags;
168*053f45beSAndroid Build Coastguard Worker 	cfg->bind_flags = usr_cfg->bind_flags;
169*053f45beSAndroid Build Coastguard Worker 
170*053f45beSAndroid Build Coastguard Worker 	return 0;
171*053f45beSAndroid Build Coastguard Worker }
172*053f45beSAndroid Build Coastguard Worker 
xsk_mmap_offsets_v1(struct xdp_mmap_offsets * off)173*053f45beSAndroid Build Coastguard Worker static void xsk_mmap_offsets_v1(struct xdp_mmap_offsets *off)
174*053f45beSAndroid Build Coastguard Worker {
175*053f45beSAndroid Build Coastguard Worker 	struct xdp_mmap_offsets_v1 off_v1;
176*053f45beSAndroid Build Coastguard Worker 
177*053f45beSAndroid Build Coastguard Worker 	/* getsockopt on a kernel <= 5.3 has no flags fields.
178*053f45beSAndroid Build Coastguard Worker 	 * Copy over the offsets to the correct places in the >=5.4 format
179*053f45beSAndroid Build Coastguard Worker 	 * and put the flags where they would have been on that kernel.
180*053f45beSAndroid Build Coastguard Worker 	 */
181*053f45beSAndroid Build Coastguard Worker 	memcpy(&off_v1, off, sizeof(off_v1));
182*053f45beSAndroid Build Coastguard Worker 
183*053f45beSAndroid Build Coastguard Worker 	off->rx.producer = off_v1.rx.producer;
184*053f45beSAndroid Build Coastguard Worker 	off->rx.consumer = off_v1.rx.consumer;
185*053f45beSAndroid Build Coastguard Worker 	off->rx.desc = off_v1.rx.desc;
186*053f45beSAndroid Build Coastguard Worker 	off->rx.flags = off_v1.rx.consumer + sizeof(__u32);
187*053f45beSAndroid Build Coastguard Worker 
188*053f45beSAndroid Build Coastguard Worker 	off->tx.producer = off_v1.tx.producer;
189*053f45beSAndroid Build Coastguard Worker 	off->tx.consumer = off_v1.tx.consumer;
190*053f45beSAndroid Build Coastguard Worker 	off->tx.desc = off_v1.tx.desc;
191*053f45beSAndroid Build Coastguard Worker 	off->tx.flags = off_v1.tx.consumer + sizeof(__u32);
192*053f45beSAndroid Build Coastguard Worker 
193*053f45beSAndroid Build Coastguard Worker 	off->fr.producer = off_v1.fr.producer;
194*053f45beSAndroid Build Coastguard Worker 	off->fr.consumer = off_v1.fr.consumer;
195*053f45beSAndroid Build Coastguard Worker 	off->fr.desc = off_v1.fr.desc;
196*053f45beSAndroid Build Coastguard Worker 	off->fr.flags = off_v1.fr.consumer + sizeof(__u32);
197*053f45beSAndroid Build Coastguard Worker 
198*053f45beSAndroid Build Coastguard Worker 	off->cr.producer = off_v1.cr.producer;
199*053f45beSAndroid Build Coastguard Worker 	off->cr.consumer = off_v1.cr.consumer;
200*053f45beSAndroid Build Coastguard Worker 	off->cr.desc = off_v1.cr.desc;
201*053f45beSAndroid Build Coastguard Worker 	off->cr.flags = off_v1.cr.consumer + sizeof(__u32);
202*053f45beSAndroid Build Coastguard Worker }
203*053f45beSAndroid Build Coastguard Worker 
xsk_get_mmap_offsets(int fd,struct xdp_mmap_offsets * off)204*053f45beSAndroid Build Coastguard Worker static int xsk_get_mmap_offsets(int fd, struct xdp_mmap_offsets *off)
205*053f45beSAndroid Build Coastguard Worker {
206*053f45beSAndroid Build Coastguard Worker 	socklen_t optlen;
207*053f45beSAndroid Build Coastguard Worker 	int err;
208*053f45beSAndroid Build Coastguard Worker 
209*053f45beSAndroid Build Coastguard Worker 	optlen = sizeof(*off);
210*053f45beSAndroid Build Coastguard Worker 	err = getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, off, &optlen);
211*053f45beSAndroid Build Coastguard Worker 	if (err)
212*053f45beSAndroid Build Coastguard Worker 		return err;
213*053f45beSAndroid Build Coastguard Worker 
214*053f45beSAndroid Build Coastguard Worker 	if (optlen == sizeof(*off))
215*053f45beSAndroid Build Coastguard Worker 		return 0;
216*053f45beSAndroid Build Coastguard Worker 
217*053f45beSAndroid Build Coastguard Worker 	if (optlen == sizeof(struct xdp_mmap_offsets_v1)) {
218*053f45beSAndroid Build Coastguard Worker 		xsk_mmap_offsets_v1(off);
219*053f45beSAndroid Build Coastguard Worker 		return 0;
220*053f45beSAndroid Build Coastguard Worker 	}
221*053f45beSAndroid Build Coastguard Worker 
222*053f45beSAndroid Build Coastguard Worker 	return -EINVAL;
223*053f45beSAndroid Build Coastguard Worker }
224*053f45beSAndroid Build Coastguard Worker 
xsk_create_umem_rings(struct xsk_umem * umem,int fd,struct xsk_ring_prod * fill,struct xsk_ring_cons * comp)225*053f45beSAndroid Build Coastguard Worker static int xsk_create_umem_rings(struct xsk_umem *umem, int fd,
226*053f45beSAndroid Build Coastguard Worker 				 struct xsk_ring_prod *fill,
227*053f45beSAndroid Build Coastguard Worker 				 struct xsk_ring_cons *comp)
228*053f45beSAndroid Build Coastguard Worker {
229*053f45beSAndroid Build Coastguard Worker 	struct xdp_mmap_offsets off;
230*053f45beSAndroid Build Coastguard Worker 	void *map;
231*053f45beSAndroid Build Coastguard Worker 	int err;
232*053f45beSAndroid Build Coastguard Worker 
233*053f45beSAndroid Build Coastguard Worker 	err = setsockopt(fd, SOL_XDP, XDP_UMEM_FILL_RING,
234*053f45beSAndroid Build Coastguard Worker 			 &umem->config.fill_size,
235*053f45beSAndroid Build Coastguard Worker 			 sizeof(umem->config.fill_size));
236*053f45beSAndroid Build Coastguard Worker 	if (err)
237*053f45beSAndroid Build Coastguard Worker 		return -errno;
238*053f45beSAndroid Build Coastguard Worker 
239*053f45beSAndroid Build Coastguard Worker 	err = setsockopt(fd, SOL_XDP, XDP_UMEM_COMPLETION_RING,
240*053f45beSAndroid Build Coastguard Worker 			 &umem->config.comp_size,
241*053f45beSAndroid Build Coastguard Worker 			 sizeof(umem->config.comp_size));
242*053f45beSAndroid Build Coastguard Worker 	if (err)
243*053f45beSAndroid Build Coastguard Worker 		return -errno;
244*053f45beSAndroid Build Coastguard Worker 
245*053f45beSAndroid Build Coastguard Worker 	err = xsk_get_mmap_offsets(fd, &off);
246*053f45beSAndroid Build Coastguard Worker 	if (err)
247*053f45beSAndroid Build Coastguard Worker 		return -errno;
248*053f45beSAndroid Build Coastguard Worker 
249*053f45beSAndroid Build Coastguard Worker 	map = mmap(NULL, off.fr.desc + umem->config.fill_size * sizeof(__u64),
250*053f45beSAndroid Build Coastguard Worker 		   PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
251*053f45beSAndroid Build Coastguard Worker 		   XDP_UMEM_PGOFF_FILL_RING);
252*053f45beSAndroid Build Coastguard Worker 	if (map == MAP_FAILED)
253*053f45beSAndroid Build Coastguard Worker 		return -errno;
254*053f45beSAndroid Build Coastguard Worker 
255*053f45beSAndroid Build Coastguard Worker 	fill->mask = umem->config.fill_size - 1;
256*053f45beSAndroid Build Coastguard Worker 	fill->size = umem->config.fill_size;
257*053f45beSAndroid Build Coastguard Worker 	fill->producer = map + off.fr.producer;
258*053f45beSAndroid Build Coastguard Worker 	fill->consumer = map + off.fr.consumer;
259*053f45beSAndroid Build Coastguard Worker 	fill->flags = map + off.fr.flags;
260*053f45beSAndroid Build Coastguard Worker 	fill->ring = map + off.fr.desc;
261*053f45beSAndroid Build Coastguard Worker 	fill->cached_cons = umem->config.fill_size;
262*053f45beSAndroid Build Coastguard Worker 
263*053f45beSAndroid Build Coastguard Worker 	map = mmap(NULL, off.cr.desc + umem->config.comp_size * sizeof(__u64),
264*053f45beSAndroid Build Coastguard Worker 		   PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
265*053f45beSAndroid Build Coastguard Worker 		   XDP_UMEM_PGOFF_COMPLETION_RING);
266*053f45beSAndroid Build Coastguard Worker 	if (map == MAP_FAILED) {
267*053f45beSAndroid Build Coastguard Worker 		err = -errno;
268*053f45beSAndroid Build Coastguard Worker 		goto out_mmap;
269*053f45beSAndroid Build Coastguard Worker 	}
270*053f45beSAndroid Build Coastguard Worker 
271*053f45beSAndroid Build Coastguard Worker 	comp->mask = umem->config.comp_size - 1;
272*053f45beSAndroid Build Coastguard Worker 	comp->size = umem->config.comp_size;
273*053f45beSAndroid Build Coastguard Worker 	comp->producer = map + off.cr.producer;
274*053f45beSAndroid Build Coastguard Worker 	comp->consumer = map + off.cr.consumer;
275*053f45beSAndroid Build Coastguard Worker 	comp->flags = map + off.cr.flags;
276*053f45beSAndroid Build Coastguard Worker 	comp->ring = map + off.cr.desc;
277*053f45beSAndroid Build Coastguard Worker 
278*053f45beSAndroid Build Coastguard Worker 	return 0;
279*053f45beSAndroid Build Coastguard Worker 
280*053f45beSAndroid Build Coastguard Worker out_mmap:
281*053f45beSAndroid Build Coastguard Worker 	munmap(map, off.fr.desc + umem->config.fill_size * sizeof(__u64));
282*053f45beSAndroid Build Coastguard Worker 	return err;
283*053f45beSAndroid Build Coastguard Worker }
284*053f45beSAndroid Build Coastguard Worker 
xsk_umem__create(struct xsk_umem ** umem_ptr,void * umem_area,__u64 size,struct xsk_ring_prod * fill,struct xsk_ring_cons * comp,const struct xsk_umem_config * usr_config)285*053f45beSAndroid Build Coastguard Worker int xsk_umem__create(struct xsk_umem **umem_ptr, void *umem_area,
286*053f45beSAndroid Build Coastguard Worker 		     __u64 size, struct xsk_ring_prod *fill,
287*053f45beSAndroid Build Coastguard Worker 		     struct xsk_ring_cons *comp,
288*053f45beSAndroid Build Coastguard Worker 		     const struct xsk_umem_config *usr_config)
289*053f45beSAndroid Build Coastguard Worker {
290*053f45beSAndroid Build Coastguard Worker 	struct xdp_umem_reg mr;
291*053f45beSAndroid Build Coastguard Worker 	struct xsk_umem *umem;
292*053f45beSAndroid Build Coastguard Worker 	int err;
293*053f45beSAndroid Build Coastguard Worker 
294*053f45beSAndroid Build Coastguard Worker 	if (!umem_area || !umem_ptr || !fill || !comp)
295*053f45beSAndroid Build Coastguard Worker 		return -EFAULT;
296*053f45beSAndroid Build Coastguard Worker 	if (!size && !xsk_page_aligned(umem_area))
297*053f45beSAndroid Build Coastguard Worker 		return -EINVAL;
298*053f45beSAndroid Build Coastguard Worker 
299*053f45beSAndroid Build Coastguard Worker 	umem = calloc(1, sizeof(*umem));
300*053f45beSAndroid Build Coastguard Worker 	if (!umem)
301*053f45beSAndroid Build Coastguard Worker 		return -ENOMEM;
302*053f45beSAndroid Build Coastguard Worker 
303*053f45beSAndroid Build Coastguard Worker 	umem->fd = socket(AF_XDP, SOCK_RAW | SOCK_CLOEXEC, 0);
304*053f45beSAndroid Build Coastguard Worker 	if (umem->fd < 0) {
305*053f45beSAndroid Build Coastguard Worker 		err = -errno;
306*053f45beSAndroid Build Coastguard Worker 		goto out_umem_alloc;
307*053f45beSAndroid Build Coastguard Worker 	}
308*053f45beSAndroid Build Coastguard Worker 
309*053f45beSAndroid Build Coastguard Worker 	umem->umem_area = umem_area;
310*053f45beSAndroid Build Coastguard Worker 	INIT_LIST_HEAD(&umem->ctx_list);
311*053f45beSAndroid Build Coastguard Worker 	xsk_set_umem_config(&umem->config, usr_config);
312*053f45beSAndroid Build Coastguard Worker 
313*053f45beSAndroid Build Coastguard Worker 	memset(&mr, 0, sizeof(mr));
314*053f45beSAndroid Build Coastguard Worker 	mr.addr = (uintptr_t)umem_area;
315*053f45beSAndroid Build Coastguard Worker 	mr.len = size;
316*053f45beSAndroid Build Coastguard Worker 	mr.chunk_size = umem->config.frame_size;
317*053f45beSAndroid Build Coastguard Worker 	mr.headroom = umem->config.frame_headroom;
318*053f45beSAndroid Build Coastguard Worker 	mr.flags = umem->config.flags;
319*053f45beSAndroid Build Coastguard Worker 
320*053f45beSAndroid Build Coastguard Worker 	err = setsockopt(umem->fd, SOL_XDP, XDP_UMEM_REG, &mr, sizeof(mr));
321*053f45beSAndroid Build Coastguard Worker 	if (err) {
322*053f45beSAndroid Build Coastguard Worker 		err = -errno;
323*053f45beSAndroid Build Coastguard Worker 		goto out_socket;
324*053f45beSAndroid Build Coastguard Worker 	}
325*053f45beSAndroid Build Coastguard Worker 
326*053f45beSAndroid Build Coastguard Worker 	err = xsk_create_umem_rings(umem, umem->fd, fill, comp);
327*053f45beSAndroid Build Coastguard Worker 	if (err)
328*053f45beSAndroid Build Coastguard Worker 		goto out_socket;
329*053f45beSAndroid Build Coastguard Worker 
330*053f45beSAndroid Build Coastguard Worker 	umem->fill_save = fill;
331*053f45beSAndroid Build Coastguard Worker 	umem->comp_save = comp;
332*053f45beSAndroid Build Coastguard Worker 	*umem_ptr = umem;
333*053f45beSAndroid Build Coastguard Worker 	return 0;
334*053f45beSAndroid Build Coastguard Worker 
335*053f45beSAndroid Build Coastguard Worker out_socket:
336*053f45beSAndroid Build Coastguard Worker 	close(umem->fd);
337*053f45beSAndroid Build Coastguard Worker out_umem_alloc:
338*053f45beSAndroid Build Coastguard Worker 	free(umem);
339*053f45beSAndroid Build Coastguard Worker 	return err;
340*053f45beSAndroid Build Coastguard Worker }
341*053f45beSAndroid Build Coastguard Worker 
342*053f45beSAndroid Build Coastguard Worker struct xsk_umem_config_v1 {
343*053f45beSAndroid Build Coastguard Worker 	__u32 fill_size;
344*053f45beSAndroid Build Coastguard Worker 	__u32 comp_size;
345*053f45beSAndroid Build Coastguard Worker 	__u32 frame_size;
346*053f45beSAndroid Build Coastguard Worker 	__u32 frame_headroom;
347*053f45beSAndroid Build Coastguard Worker };
348*053f45beSAndroid Build Coastguard Worker 
get_xsk_prog(void)349*053f45beSAndroid Build Coastguard Worker static enum xsk_prog get_xsk_prog(void)
350*053f45beSAndroid Build Coastguard Worker {
351*053f45beSAndroid Build Coastguard Worker 	enum xsk_prog detected = XSK_PROG_FALLBACK;
352*053f45beSAndroid Build Coastguard Worker 	char data_in = 0, data_out;
353*053f45beSAndroid Build Coastguard Worker 	struct bpf_insn insns[] = {
354*053f45beSAndroid Build Coastguard Worker 		BPF_LD_MAP_FD(BPF_REG_1, 0),
355*053f45beSAndroid Build Coastguard Worker 		BPF_MOV64_IMM(BPF_REG_2, 0),
356*053f45beSAndroid Build Coastguard Worker 		BPF_MOV64_IMM(BPF_REG_3, XDP_PASS),
357*053f45beSAndroid Build Coastguard Worker 		BPF_EMIT_CALL(BPF_FUNC_redirect_map),
358*053f45beSAndroid Build Coastguard Worker 		BPF_EXIT_INSN(),
359*053f45beSAndroid Build Coastguard Worker 	};
360*053f45beSAndroid Build Coastguard Worker 	LIBBPF_OPTS(bpf_test_run_opts, opts,
361*053f45beSAndroid Build Coastguard Worker 		.data_in = &data_in,
362*053f45beSAndroid Build Coastguard Worker 		.data_size_in = 1,
363*053f45beSAndroid Build Coastguard Worker 		.data_out = &data_out,
364*053f45beSAndroid Build Coastguard Worker 	);
365*053f45beSAndroid Build Coastguard Worker 
366*053f45beSAndroid Build Coastguard Worker 	int prog_fd, map_fd, ret, insn_cnt = ARRAY_SIZE(insns);
367*053f45beSAndroid Build Coastguard Worker 
368*053f45beSAndroid Build Coastguard Worker 	map_fd = bpf_map_create(BPF_MAP_TYPE_XSKMAP, NULL, sizeof(int), sizeof(int), 1, NULL);
369*053f45beSAndroid Build Coastguard Worker 	if (map_fd < 0)
370*053f45beSAndroid Build Coastguard Worker 		return detected;
371*053f45beSAndroid Build Coastguard Worker 
372*053f45beSAndroid Build Coastguard Worker 	insns[0].imm = map_fd;
373*053f45beSAndroid Build Coastguard Worker 
374*053f45beSAndroid Build Coastguard Worker 	prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL);
375*053f45beSAndroid Build Coastguard Worker 	if (prog_fd < 0) {
376*053f45beSAndroid Build Coastguard Worker 		close(map_fd);
377*053f45beSAndroid Build Coastguard Worker 		return detected;
378*053f45beSAndroid Build Coastguard Worker 	}
379*053f45beSAndroid Build Coastguard Worker 
380*053f45beSAndroid Build Coastguard Worker 	ret = bpf_prog_test_run_opts(prog_fd, &opts);
381*053f45beSAndroid Build Coastguard Worker 	if (!ret && opts.retval == XDP_PASS)
382*053f45beSAndroid Build Coastguard Worker 		detected = XSK_PROG_REDIRECT_FLAGS;
383*053f45beSAndroid Build Coastguard Worker 	close(prog_fd);
384*053f45beSAndroid Build Coastguard Worker 	close(map_fd);
385*053f45beSAndroid Build Coastguard Worker 	return detected;
386*053f45beSAndroid Build Coastguard Worker }
387*053f45beSAndroid Build Coastguard Worker 
xsk_load_xdp_prog(struct xsk_socket * xsk)388*053f45beSAndroid Build Coastguard Worker static int xsk_load_xdp_prog(struct xsk_socket *xsk)
389*053f45beSAndroid Build Coastguard Worker {
390*053f45beSAndroid Build Coastguard Worker 	static const int log_buf_size = 16 * 1024;
391*053f45beSAndroid Build Coastguard Worker 	struct xsk_ctx *ctx = xsk->ctx;
392*053f45beSAndroid Build Coastguard Worker 	char log_buf[log_buf_size];
393*053f45beSAndroid Build Coastguard Worker 	int prog_fd;
394*053f45beSAndroid Build Coastguard Worker 
395*053f45beSAndroid Build Coastguard Worker 	/* This is the fallback C-program:
396*053f45beSAndroid Build Coastguard Worker 	 * SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)
397*053f45beSAndroid Build Coastguard Worker 	 * {
398*053f45beSAndroid Build Coastguard Worker 	 *     int ret, index = ctx->rx_queue_index;
399*053f45beSAndroid Build Coastguard Worker 	 *
400*053f45beSAndroid Build Coastguard Worker 	 *     // A set entry here means that the correspnding queue_id
401*053f45beSAndroid Build Coastguard Worker 	 *     // has an active AF_XDP socket bound to it.
402*053f45beSAndroid Build Coastguard Worker 	 *     ret = bpf_redirect_map(&xsks_map, index, XDP_PASS);
403*053f45beSAndroid Build Coastguard Worker 	 *     if (ret > 0)
404*053f45beSAndroid Build Coastguard Worker 	 *         return ret;
405*053f45beSAndroid Build Coastguard Worker 	 *
406*053f45beSAndroid Build Coastguard Worker 	 *     // Fallback for pre-5.3 kernels, not supporting default
407*053f45beSAndroid Build Coastguard Worker 	 *     // action in the flags parameter.
408*053f45beSAndroid Build Coastguard Worker 	 *     if (bpf_map_lookup_elem(&xsks_map, &index))
409*053f45beSAndroid Build Coastguard Worker 	 *         return bpf_redirect_map(&xsks_map, index, 0);
410*053f45beSAndroid Build Coastguard Worker 	 *     return XDP_PASS;
411*053f45beSAndroid Build Coastguard Worker 	 * }
412*053f45beSAndroid Build Coastguard Worker 	 */
413*053f45beSAndroid Build Coastguard Worker 	struct bpf_insn prog[] = {
414*053f45beSAndroid Build Coastguard Worker 		/* r2 = *(u32 *)(r1 + 16) */
415*053f45beSAndroid Build Coastguard Worker 		BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 16),
416*053f45beSAndroid Build Coastguard Worker 		/* *(u32 *)(r10 - 4) = r2 */
417*053f45beSAndroid Build Coastguard Worker 		BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_2, -4),
418*053f45beSAndroid Build Coastguard Worker 		/* r1 = xskmap[] */
419*053f45beSAndroid Build Coastguard Worker 		BPF_LD_MAP_FD(BPF_REG_1, ctx->xsks_map_fd),
420*053f45beSAndroid Build Coastguard Worker 		/* r3 = XDP_PASS */
421*053f45beSAndroid Build Coastguard Worker 		BPF_MOV64_IMM(BPF_REG_3, 2),
422*053f45beSAndroid Build Coastguard Worker 		/* call bpf_redirect_map */
423*053f45beSAndroid Build Coastguard Worker 		BPF_EMIT_CALL(BPF_FUNC_redirect_map),
424*053f45beSAndroid Build Coastguard Worker 		/* if w0 != 0 goto pc+13 */
425*053f45beSAndroid Build Coastguard Worker 		BPF_JMP32_IMM(BPF_JSGT, BPF_REG_0, 0, 13),
426*053f45beSAndroid Build Coastguard Worker 		/* r2 = r10 */
427*053f45beSAndroid Build Coastguard Worker 		BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
428*053f45beSAndroid Build Coastguard Worker 		/* r2 += -4 */
429*053f45beSAndroid Build Coastguard Worker 		BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4),
430*053f45beSAndroid Build Coastguard Worker 		/* r1 = xskmap[] */
431*053f45beSAndroid Build Coastguard Worker 		BPF_LD_MAP_FD(BPF_REG_1, ctx->xsks_map_fd),
432*053f45beSAndroid Build Coastguard Worker 		/* call bpf_map_lookup_elem */
433*053f45beSAndroid Build Coastguard Worker 		BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
434*053f45beSAndroid Build Coastguard Worker 		/* r1 = r0 */
435*053f45beSAndroid Build Coastguard Worker 		BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
436*053f45beSAndroid Build Coastguard Worker 		/* r0 = XDP_PASS */
437*053f45beSAndroid Build Coastguard Worker 		BPF_MOV64_IMM(BPF_REG_0, 2),
438*053f45beSAndroid Build Coastguard Worker 		/* if r1 == 0 goto pc+5 */
439*053f45beSAndroid Build Coastguard Worker 		BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 5),
440*053f45beSAndroid Build Coastguard Worker 		/* r2 = *(u32 *)(r10 - 4) */
441*053f45beSAndroid Build Coastguard Worker 		BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_10, -4),
442*053f45beSAndroid Build Coastguard Worker 		/* r1 = xskmap[] */
443*053f45beSAndroid Build Coastguard Worker 		BPF_LD_MAP_FD(BPF_REG_1, ctx->xsks_map_fd),
444*053f45beSAndroid Build Coastguard Worker 		/* r3 = 0 */
445*053f45beSAndroid Build Coastguard Worker 		BPF_MOV64_IMM(BPF_REG_3, 0),
446*053f45beSAndroid Build Coastguard Worker 		/* call bpf_redirect_map */
447*053f45beSAndroid Build Coastguard Worker 		BPF_EMIT_CALL(BPF_FUNC_redirect_map),
448*053f45beSAndroid Build Coastguard Worker 		/* The jumps are to this instruction */
449*053f45beSAndroid Build Coastguard Worker 		BPF_EXIT_INSN(),
450*053f45beSAndroid Build Coastguard Worker 	};
451*053f45beSAndroid Build Coastguard Worker 
452*053f45beSAndroid Build Coastguard Worker 	/* This is the post-5.3 kernel C-program:
453*053f45beSAndroid Build Coastguard Worker 	 * SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)
454*053f45beSAndroid Build Coastguard Worker 	 * {
455*053f45beSAndroid Build Coastguard Worker 	 *     return bpf_redirect_map(&xsks_map, ctx->rx_queue_index, XDP_PASS);
456*053f45beSAndroid Build Coastguard Worker 	 * }
457*053f45beSAndroid Build Coastguard Worker 	 */
458*053f45beSAndroid Build Coastguard Worker 	struct bpf_insn prog_redirect_flags[] = {
459*053f45beSAndroid Build Coastguard Worker 		/* r2 = *(u32 *)(r1 + 16) */
460*053f45beSAndroid Build Coastguard Worker 		BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 16),
461*053f45beSAndroid Build Coastguard Worker 		/* r1 = xskmap[] */
462*053f45beSAndroid Build Coastguard Worker 		BPF_LD_MAP_FD(BPF_REG_1, ctx->xsks_map_fd),
463*053f45beSAndroid Build Coastguard Worker 		/* r3 = XDP_PASS */
464*053f45beSAndroid Build Coastguard Worker 		BPF_MOV64_IMM(BPF_REG_3, 2),
465*053f45beSAndroid Build Coastguard Worker 		/* call bpf_redirect_map */
466*053f45beSAndroid Build Coastguard Worker 		BPF_EMIT_CALL(BPF_FUNC_redirect_map),
467*053f45beSAndroid Build Coastguard Worker 		BPF_EXIT_INSN(),
468*053f45beSAndroid Build Coastguard Worker 	};
469*053f45beSAndroid Build Coastguard Worker 	size_t insns_cnt[] = {ARRAY_SIZE(prog),
470*053f45beSAndroid Build Coastguard Worker 			      ARRAY_SIZE(prog_redirect_flags),
471*053f45beSAndroid Build Coastguard Worker 	};
472*053f45beSAndroid Build Coastguard Worker 	struct bpf_insn *progs[] = {prog, prog_redirect_flags};
473*053f45beSAndroid Build Coastguard Worker 	enum xsk_prog option = get_xsk_prog();
474*053f45beSAndroid Build Coastguard Worker 	LIBBPF_OPTS(bpf_prog_load_opts, opts,
475*053f45beSAndroid Build Coastguard Worker 		.log_buf = log_buf,
476*053f45beSAndroid Build Coastguard Worker 		.log_size = log_buf_size,
477*053f45beSAndroid Build Coastguard Worker 	);
478*053f45beSAndroid Build Coastguard Worker 
479*053f45beSAndroid Build Coastguard Worker 	prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "LGPL-2.1 or BSD-2-Clause",
480*053f45beSAndroid Build Coastguard Worker 				progs[option], insns_cnt[option], &opts);
481*053f45beSAndroid Build Coastguard Worker 	if (prog_fd < 0) {
482*053f45beSAndroid Build Coastguard Worker 		pr_warn("BPF log buffer:\n%s", log_buf);
483*053f45beSAndroid Build Coastguard Worker 		return prog_fd;
484*053f45beSAndroid Build Coastguard Worker 	}
485*053f45beSAndroid Build Coastguard Worker 
486*053f45beSAndroid Build Coastguard Worker 	ctx->prog_fd = prog_fd;
487*053f45beSAndroid Build Coastguard Worker 	return 0;
488*053f45beSAndroid Build Coastguard Worker }
489*053f45beSAndroid Build Coastguard Worker 
xsk_create_bpf_link(struct xsk_socket * xsk)490*053f45beSAndroid Build Coastguard Worker static int xsk_create_bpf_link(struct xsk_socket *xsk)
491*053f45beSAndroid Build Coastguard Worker {
492*053f45beSAndroid Build Coastguard Worker 	DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts);
493*053f45beSAndroid Build Coastguard Worker 	struct xsk_ctx *ctx = xsk->ctx;
494*053f45beSAndroid Build Coastguard Worker 	__u32 prog_id = 0;
495*053f45beSAndroid Build Coastguard Worker 	int link_fd;
496*053f45beSAndroid Build Coastguard Worker 	int err;
497*053f45beSAndroid Build Coastguard Worker 
498*053f45beSAndroid Build Coastguard Worker 	err = bpf_xdp_query_id(ctx->ifindex, xsk->config.xdp_flags, &prog_id);
499*053f45beSAndroid Build Coastguard Worker 	if (err) {
500*053f45beSAndroid Build Coastguard Worker 		pr_warn("getting XDP prog id failed\n");
501*053f45beSAndroid Build Coastguard Worker 		return err;
502*053f45beSAndroid Build Coastguard Worker 	}
503*053f45beSAndroid Build Coastguard Worker 
504*053f45beSAndroid Build Coastguard Worker 	/* if there's a netlink-based XDP prog loaded on interface, bail out
505*053f45beSAndroid Build Coastguard Worker 	 * and ask user to do the removal by himself
506*053f45beSAndroid Build Coastguard Worker 	 */
507*053f45beSAndroid Build Coastguard Worker 	if (prog_id) {
508*053f45beSAndroid Build Coastguard Worker 		pr_warn("Netlink-based XDP prog detected, please unload it in order to launch AF_XDP prog\n");
509*053f45beSAndroid Build Coastguard Worker 		return -EINVAL;
510*053f45beSAndroid Build Coastguard Worker 	}
511*053f45beSAndroid Build Coastguard Worker 
512*053f45beSAndroid Build Coastguard Worker 	opts.flags = xsk->config.xdp_flags & ~(XDP_FLAGS_UPDATE_IF_NOEXIST | XDP_FLAGS_REPLACE);
513*053f45beSAndroid Build Coastguard Worker 
514*053f45beSAndroid Build Coastguard Worker 	link_fd = bpf_link_create(ctx->prog_fd, ctx->ifindex, BPF_XDP, &opts);
515*053f45beSAndroid Build Coastguard Worker 	if (link_fd < 0) {
516*053f45beSAndroid Build Coastguard Worker 		pr_warn("bpf_link_create failed: %s\n", strerror(errno));
517*053f45beSAndroid Build Coastguard Worker 		return link_fd;
518*053f45beSAndroid Build Coastguard Worker 	}
519*053f45beSAndroid Build Coastguard Worker 
520*053f45beSAndroid Build Coastguard Worker 	ctx->link_fd = link_fd;
521*053f45beSAndroid Build Coastguard Worker 	return 0;
522*053f45beSAndroid Build Coastguard Worker }
523*053f45beSAndroid Build Coastguard Worker 
524*053f45beSAndroid Build Coastguard Worker /* Copy up to sz - 1 bytes from zero-terminated src string and ensure that dst
525*053f45beSAndroid Build Coastguard Worker  * is zero-terminated string no matter what (unless sz == 0, in which case
526*053f45beSAndroid Build Coastguard Worker  * it's a no-op). It's conceptually close to FreeBSD's strlcpy(), but differs
527*053f45beSAndroid Build Coastguard Worker  * in what is returned. Given this is internal helper, it's trivial to extend
528*053f45beSAndroid Build Coastguard Worker  * this, when necessary. Use this instead of strncpy inside libbpf source code.
529*053f45beSAndroid Build Coastguard Worker  */
libbpf_strlcpy(char * dst,const char * src,size_t sz)530*053f45beSAndroid Build Coastguard Worker static inline void libbpf_strlcpy(char *dst, const char *src, size_t sz)
531*053f45beSAndroid Build Coastguard Worker {
532*053f45beSAndroid Build Coastguard Worker         size_t i;
533*053f45beSAndroid Build Coastguard Worker 
534*053f45beSAndroid Build Coastguard Worker         if (sz == 0)
535*053f45beSAndroid Build Coastguard Worker                 return;
536*053f45beSAndroid Build Coastguard Worker 
537*053f45beSAndroid Build Coastguard Worker         sz--;
538*053f45beSAndroid Build Coastguard Worker         for (i = 0; i < sz && src[i]; i++)
539*053f45beSAndroid Build Coastguard Worker                 dst[i] = src[i];
540*053f45beSAndroid Build Coastguard Worker         dst[i] = '\0';
541*053f45beSAndroid Build Coastguard Worker }
542*053f45beSAndroid Build Coastguard Worker 
xsk_get_max_queues(struct xsk_socket * xsk)543*053f45beSAndroid Build Coastguard Worker static int xsk_get_max_queues(struct xsk_socket *xsk)
544*053f45beSAndroid Build Coastguard Worker {
545*053f45beSAndroid Build Coastguard Worker 	struct ethtool_channels channels = { .cmd = ETHTOOL_GCHANNELS };
546*053f45beSAndroid Build Coastguard Worker 	struct xsk_ctx *ctx = xsk->ctx;
547*053f45beSAndroid Build Coastguard Worker 	struct ifreq ifr = {};
548*053f45beSAndroid Build Coastguard Worker 	int fd, err, ret;
549*053f45beSAndroid Build Coastguard Worker 
550*053f45beSAndroid Build Coastguard Worker 	fd = socket(AF_LOCAL, SOCK_DGRAM | SOCK_CLOEXEC, 0);
551*053f45beSAndroid Build Coastguard Worker 	if (fd < 0)
552*053f45beSAndroid Build Coastguard Worker 		return -errno;
553*053f45beSAndroid Build Coastguard Worker 
554*053f45beSAndroid Build Coastguard Worker 	ifr.ifr_data = (void *)&channels;
555*053f45beSAndroid Build Coastguard Worker 	libbpf_strlcpy(ifr.ifr_name, ctx->ifname, IFNAMSIZ);
556*053f45beSAndroid Build Coastguard Worker 	err = ioctl(fd, SIOCETHTOOL, &ifr);
557*053f45beSAndroid Build Coastguard Worker 	if (err && errno != EOPNOTSUPP) {
558*053f45beSAndroid Build Coastguard Worker 		ret = -errno;
559*053f45beSAndroid Build Coastguard Worker 		goto out;
560*053f45beSAndroid Build Coastguard Worker 	}
561*053f45beSAndroid Build Coastguard Worker 
562*053f45beSAndroid Build Coastguard Worker 	if (err) {
563*053f45beSAndroid Build Coastguard Worker 		/* If the device says it has no channels, then all traffic
564*053f45beSAndroid Build Coastguard Worker 		 * is sent to a single stream, so max queues = 1.
565*053f45beSAndroid Build Coastguard Worker 		 */
566*053f45beSAndroid Build Coastguard Worker 		ret = 1;
567*053f45beSAndroid Build Coastguard Worker 	} else {
568*053f45beSAndroid Build Coastguard Worker 		/* Take the max of rx, tx, combined. Drivers return
569*053f45beSAndroid Build Coastguard Worker 		 * the number of channels in different ways.
570*053f45beSAndroid Build Coastguard Worker 		 */
571*053f45beSAndroid Build Coastguard Worker 		ret = max(channels.max_rx, channels.max_tx);
572*053f45beSAndroid Build Coastguard Worker 		ret = max(ret, (int)channels.max_combined);
573*053f45beSAndroid Build Coastguard Worker 	}
574*053f45beSAndroid Build Coastguard Worker 
575*053f45beSAndroid Build Coastguard Worker out:
576*053f45beSAndroid Build Coastguard Worker 	close(fd);
577*053f45beSAndroid Build Coastguard Worker 	return ret;
578*053f45beSAndroid Build Coastguard Worker }
579*053f45beSAndroid Build Coastguard Worker 
xsk_create_bpf_maps(struct xsk_socket * xsk)580*053f45beSAndroid Build Coastguard Worker static int xsk_create_bpf_maps(struct xsk_socket *xsk)
581*053f45beSAndroid Build Coastguard Worker {
582*053f45beSAndroid Build Coastguard Worker 	struct xsk_ctx *ctx = xsk->ctx;
583*053f45beSAndroid Build Coastguard Worker 	int max_queues;
584*053f45beSAndroid Build Coastguard Worker 	int fd;
585*053f45beSAndroid Build Coastguard Worker 
586*053f45beSAndroid Build Coastguard Worker 	max_queues = xsk_get_max_queues(xsk);
587*053f45beSAndroid Build Coastguard Worker 	if (max_queues < 0)
588*053f45beSAndroid Build Coastguard Worker 		return max_queues;
589*053f45beSAndroid Build Coastguard Worker 
590*053f45beSAndroid Build Coastguard Worker 	fd = bpf_map_create(BPF_MAP_TYPE_XSKMAP, "xsks_map",
591*053f45beSAndroid Build Coastguard Worker 			    sizeof(int), sizeof(int), max_queues, NULL);
592*053f45beSAndroid Build Coastguard Worker 	if (fd < 0)
593*053f45beSAndroid Build Coastguard Worker 		return fd;
594*053f45beSAndroid Build Coastguard Worker 
595*053f45beSAndroid Build Coastguard Worker 	ctx->xsks_map_fd = fd;
596*053f45beSAndroid Build Coastguard Worker 
597*053f45beSAndroid Build Coastguard Worker 	return 0;
598*053f45beSAndroid Build Coastguard Worker }
599*053f45beSAndroid Build Coastguard Worker 
xsk_delete_bpf_maps(struct xsk_socket * xsk)600*053f45beSAndroid Build Coastguard Worker static void xsk_delete_bpf_maps(struct xsk_socket *xsk)
601*053f45beSAndroid Build Coastguard Worker {
602*053f45beSAndroid Build Coastguard Worker 	struct xsk_ctx *ctx = xsk->ctx;
603*053f45beSAndroid Build Coastguard Worker 
604*053f45beSAndroid Build Coastguard Worker 	bpf_map_delete_elem(ctx->xsks_map_fd, &ctx->queue_id);
605*053f45beSAndroid Build Coastguard Worker 	close(ctx->xsks_map_fd);
606*053f45beSAndroid Build Coastguard Worker }
607*053f45beSAndroid Build Coastguard Worker 
xsk_lookup_bpf_maps(struct xsk_socket * xsk)608*053f45beSAndroid Build Coastguard Worker static int xsk_lookup_bpf_maps(struct xsk_socket *xsk)
609*053f45beSAndroid Build Coastguard Worker {
610*053f45beSAndroid Build Coastguard Worker 	__u32 i, *map_ids, num_maps, prog_len = sizeof(struct bpf_prog_info);
611*053f45beSAndroid Build Coastguard Worker 	__u32 map_len = sizeof(struct bpf_map_info);
612*053f45beSAndroid Build Coastguard Worker 	struct bpf_prog_info prog_info = {};
613*053f45beSAndroid Build Coastguard Worker 	struct xsk_ctx *ctx = xsk->ctx;
614*053f45beSAndroid Build Coastguard Worker 	struct bpf_map_info map_info;
615*053f45beSAndroid Build Coastguard Worker 	int fd, err;
616*053f45beSAndroid Build Coastguard Worker 
617*053f45beSAndroid Build Coastguard Worker 	err = bpf_obj_get_info_by_fd(ctx->prog_fd, &prog_info, &prog_len);
618*053f45beSAndroid Build Coastguard Worker 	if (err)
619*053f45beSAndroid Build Coastguard Worker 		return err;
620*053f45beSAndroid Build Coastguard Worker 
621*053f45beSAndroid Build Coastguard Worker 	num_maps = prog_info.nr_map_ids;
622*053f45beSAndroid Build Coastguard Worker 
623*053f45beSAndroid Build Coastguard Worker 	map_ids = calloc(prog_info.nr_map_ids, sizeof(*map_ids));
624*053f45beSAndroid Build Coastguard Worker 	if (!map_ids)
625*053f45beSAndroid Build Coastguard Worker 		return -ENOMEM;
626*053f45beSAndroid Build Coastguard Worker 
627*053f45beSAndroid Build Coastguard Worker 	memset(&prog_info, 0, prog_len);
628*053f45beSAndroid Build Coastguard Worker 	prog_info.nr_map_ids = num_maps;
629*053f45beSAndroid Build Coastguard Worker 	prog_info.map_ids = (__u64)(unsigned long)map_ids;
630*053f45beSAndroid Build Coastguard Worker 
631*053f45beSAndroid Build Coastguard Worker 	err = bpf_obj_get_info_by_fd(ctx->prog_fd, &prog_info, &prog_len);
632*053f45beSAndroid Build Coastguard Worker 	if (err)
633*053f45beSAndroid Build Coastguard Worker 		goto out_map_ids;
634*053f45beSAndroid Build Coastguard Worker 
635*053f45beSAndroid Build Coastguard Worker 	ctx->xsks_map_fd = -1;
636*053f45beSAndroid Build Coastguard Worker 
637*053f45beSAndroid Build Coastguard Worker 	for (i = 0; i < prog_info.nr_map_ids; i++) {
638*053f45beSAndroid Build Coastguard Worker 		fd = bpf_map_get_fd_by_id(map_ids[i]);
639*053f45beSAndroid Build Coastguard Worker 		if (fd < 0)
640*053f45beSAndroid Build Coastguard Worker 			continue;
641*053f45beSAndroid Build Coastguard Worker 
642*053f45beSAndroid Build Coastguard Worker 		memset(&map_info, 0, map_len);
643*053f45beSAndroid Build Coastguard Worker 		err = bpf_obj_get_info_by_fd(fd, &map_info, &map_len);
644*053f45beSAndroid Build Coastguard Worker 		if (err) {
645*053f45beSAndroid Build Coastguard Worker 			close(fd);
646*053f45beSAndroid Build Coastguard Worker 			continue;
647*053f45beSAndroid Build Coastguard Worker 		}
648*053f45beSAndroid Build Coastguard Worker 
649*053f45beSAndroid Build Coastguard Worker 		if (!strncmp(map_info.name, "xsks_map", sizeof(map_info.name))) {
650*053f45beSAndroid Build Coastguard Worker 			ctx->xsks_map_fd = fd;
651*053f45beSAndroid Build Coastguard Worker 			break;
652*053f45beSAndroid Build Coastguard Worker 		}
653*053f45beSAndroid Build Coastguard Worker 
654*053f45beSAndroid Build Coastguard Worker 		close(fd);
655*053f45beSAndroid Build Coastguard Worker 	}
656*053f45beSAndroid Build Coastguard Worker 
657*053f45beSAndroid Build Coastguard Worker 	if (ctx->xsks_map_fd == -1)
658*053f45beSAndroid Build Coastguard Worker 		err = -ENOENT;
659*053f45beSAndroid Build Coastguard Worker 
660*053f45beSAndroid Build Coastguard Worker out_map_ids:
661*053f45beSAndroid Build Coastguard Worker 	free(map_ids);
662*053f45beSAndroid Build Coastguard Worker 	return err;
663*053f45beSAndroid Build Coastguard Worker }
664*053f45beSAndroid Build Coastguard Worker 
xsk_set_bpf_maps(struct xsk_socket * xsk)665*053f45beSAndroid Build Coastguard Worker static int xsk_set_bpf_maps(struct xsk_socket *xsk)
666*053f45beSAndroid Build Coastguard Worker {
667*053f45beSAndroid Build Coastguard Worker 	struct xsk_ctx *ctx = xsk->ctx;
668*053f45beSAndroid Build Coastguard Worker 
669*053f45beSAndroid Build Coastguard Worker 	return bpf_map_update_elem(ctx->xsks_map_fd, &ctx->queue_id,
670*053f45beSAndroid Build Coastguard Worker 				   &xsk->fd, 0);
671*053f45beSAndroid Build Coastguard Worker }
672*053f45beSAndroid Build Coastguard Worker 
xsk_link_lookup(int ifindex,__u32 * prog_id,int * link_fd)673*053f45beSAndroid Build Coastguard Worker static int xsk_link_lookup(int ifindex, __u32 *prog_id, int *link_fd)
674*053f45beSAndroid Build Coastguard Worker {
675*053f45beSAndroid Build Coastguard Worker 	struct bpf_link_info link_info;
676*053f45beSAndroid Build Coastguard Worker 	__u32 link_len;
677*053f45beSAndroid Build Coastguard Worker 	__u32 id = 0;
678*053f45beSAndroid Build Coastguard Worker 	int err;
679*053f45beSAndroid Build Coastguard Worker 	int fd;
680*053f45beSAndroid Build Coastguard Worker 
681*053f45beSAndroid Build Coastguard Worker 	while (true) {
682*053f45beSAndroid Build Coastguard Worker 		err = bpf_link_get_next_id(id, &id);
683*053f45beSAndroid Build Coastguard Worker 		if (err) {
684*053f45beSAndroid Build Coastguard Worker 			if (errno == ENOENT) {
685*053f45beSAndroid Build Coastguard Worker 				err = 0;
686*053f45beSAndroid Build Coastguard Worker 				break;
687*053f45beSAndroid Build Coastguard Worker 			}
688*053f45beSAndroid Build Coastguard Worker 			pr_warn("can't get next link: %s\n", strerror(errno));
689*053f45beSAndroid Build Coastguard Worker 			break;
690*053f45beSAndroid Build Coastguard Worker 		}
691*053f45beSAndroid Build Coastguard Worker 
692*053f45beSAndroid Build Coastguard Worker 		fd = bpf_link_get_fd_by_id(id);
693*053f45beSAndroid Build Coastguard Worker 		if (fd < 0) {
694*053f45beSAndroid Build Coastguard Worker 			if (errno == ENOENT)
695*053f45beSAndroid Build Coastguard Worker 				continue;
696*053f45beSAndroid Build Coastguard Worker 			pr_warn("can't get link by id (%u): %s\n", id, strerror(errno));
697*053f45beSAndroid Build Coastguard Worker 			err = -errno;
698*053f45beSAndroid Build Coastguard Worker 			break;
699*053f45beSAndroid Build Coastguard Worker 		}
700*053f45beSAndroid Build Coastguard Worker 
701*053f45beSAndroid Build Coastguard Worker 		link_len = sizeof(struct bpf_link_info);
702*053f45beSAndroid Build Coastguard Worker 		memset(&link_info, 0, link_len);
703*053f45beSAndroid Build Coastguard Worker 		err = bpf_obj_get_info_by_fd(fd, &link_info, &link_len);
704*053f45beSAndroid Build Coastguard Worker 		if (err) {
705*053f45beSAndroid Build Coastguard Worker 			pr_warn("can't get link info: %s\n", strerror(errno));
706*053f45beSAndroid Build Coastguard Worker 			close(fd);
707*053f45beSAndroid Build Coastguard Worker 			break;
708*053f45beSAndroid Build Coastguard Worker 		}
709*053f45beSAndroid Build Coastguard Worker 		if (link_info.type == BPF_LINK_TYPE_XDP) {
710*053f45beSAndroid Build Coastguard Worker 			if (link_info.xdp.ifindex == ifindex) {
711*053f45beSAndroid Build Coastguard Worker 				*link_fd = fd;
712*053f45beSAndroid Build Coastguard Worker 				if (prog_id)
713*053f45beSAndroid Build Coastguard Worker 					*prog_id = link_info.prog_id;
714*053f45beSAndroid Build Coastguard Worker 				break;
715*053f45beSAndroid Build Coastguard Worker 			}
716*053f45beSAndroid Build Coastguard Worker 		}
717*053f45beSAndroid Build Coastguard Worker 		close(fd);
718*053f45beSAndroid Build Coastguard Worker 	}
719*053f45beSAndroid Build Coastguard Worker 
720*053f45beSAndroid Build Coastguard Worker 	return err;
721*053f45beSAndroid Build Coastguard Worker }
722*053f45beSAndroid Build Coastguard Worker 
xsk_probe_bpf_link(void)723*053f45beSAndroid Build Coastguard Worker static bool xsk_probe_bpf_link(void)
724*053f45beSAndroid Build Coastguard Worker {
725*053f45beSAndroid Build Coastguard Worker 	LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = XDP_FLAGS_SKB_MODE);
726*053f45beSAndroid Build Coastguard Worker 	struct bpf_insn insns[2] = {
727*053f45beSAndroid Build Coastguard Worker 		BPF_MOV64_IMM(BPF_REG_0, XDP_PASS),
728*053f45beSAndroid Build Coastguard Worker 		BPF_EXIT_INSN()
729*053f45beSAndroid Build Coastguard Worker 	};
730*053f45beSAndroid Build Coastguard Worker 	int prog_fd, link_fd = -1, insn_cnt = ARRAY_SIZE(insns);
731*053f45beSAndroid Build Coastguard Worker 	int ifindex_lo = 1;
732*053f45beSAndroid Build Coastguard Worker 	bool ret = false;
733*053f45beSAndroid Build Coastguard Worker 	int err;
734*053f45beSAndroid Build Coastguard Worker 
735*053f45beSAndroid Build Coastguard Worker 	err = xsk_link_lookup(ifindex_lo, NULL, &link_fd);
736*053f45beSAndroid Build Coastguard Worker 	if (err)
737*053f45beSAndroid Build Coastguard Worker 		return ret;
738*053f45beSAndroid Build Coastguard Worker 
739*053f45beSAndroid Build Coastguard Worker 	if (link_fd >= 0)
740*053f45beSAndroid Build Coastguard Worker 		return true;
741*053f45beSAndroid Build Coastguard Worker 
742*053f45beSAndroid Build Coastguard Worker 	prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL);
743*053f45beSAndroid Build Coastguard Worker 	if (prog_fd < 0)
744*053f45beSAndroid Build Coastguard Worker 		return ret;
745*053f45beSAndroid Build Coastguard Worker 
746*053f45beSAndroid Build Coastguard Worker 	link_fd = bpf_link_create(prog_fd, ifindex_lo, BPF_XDP, &opts);
747*053f45beSAndroid Build Coastguard Worker 	close(prog_fd);
748*053f45beSAndroid Build Coastguard Worker 
749*053f45beSAndroid Build Coastguard Worker 	if (link_fd >= 0) {
750*053f45beSAndroid Build Coastguard Worker 		ret = true;
751*053f45beSAndroid Build Coastguard Worker 		close(link_fd);
752*053f45beSAndroid Build Coastguard Worker 	}
753*053f45beSAndroid Build Coastguard Worker 
754*053f45beSAndroid Build Coastguard Worker 	return ret;
755*053f45beSAndroid Build Coastguard Worker }
756*053f45beSAndroid Build Coastguard Worker 
xsk_create_xsk_struct(int ifindex,struct xsk_socket * xsk)757*053f45beSAndroid Build Coastguard Worker static int xsk_create_xsk_struct(int ifindex, struct xsk_socket *xsk)
758*053f45beSAndroid Build Coastguard Worker {
759*053f45beSAndroid Build Coastguard Worker 	char ifname[IFNAMSIZ];
760*053f45beSAndroid Build Coastguard Worker 	struct xsk_ctx *ctx;
761*053f45beSAndroid Build Coastguard Worker 	char *interface;
762*053f45beSAndroid Build Coastguard Worker 
763*053f45beSAndroid Build Coastguard Worker 	ctx = calloc(1, sizeof(*ctx));
764*053f45beSAndroid Build Coastguard Worker 	if (!ctx)
765*053f45beSAndroid Build Coastguard Worker 		return -ENOMEM;
766*053f45beSAndroid Build Coastguard Worker 
767*053f45beSAndroid Build Coastguard Worker 	interface = if_indextoname(ifindex, &ifname[0]);
768*053f45beSAndroid Build Coastguard Worker 	if (!interface) {
769*053f45beSAndroid Build Coastguard Worker 		free(ctx);
770*053f45beSAndroid Build Coastguard Worker 		return -errno;
771*053f45beSAndroid Build Coastguard Worker 	}
772*053f45beSAndroid Build Coastguard Worker 
773*053f45beSAndroid Build Coastguard Worker 	ctx->ifindex = ifindex;
774*053f45beSAndroid Build Coastguard Worker 	libbpf_strlcpy(ctx->ifname, ifname, IFNAMSIZ);
775*053f45beSAndroid Build Coastguard Worker 
776*053f45beSAndroid Build Coastguard Worker 	xsk->ctx = ctx;
777*053f45beSAndroid Build Coastguard Worker 	xsk->ctx->has_bpf_link = xsk_probe_bpf_link();
778*053f45beSAndroid Build Coastguard Worker 
779*053f45beSAndroid Build Coastguard Worker 	return 0;
780*053f45beSAndroid Build Coastguard Worker }
781*053f45beSAndroid Build Coastguard Worker 
xsk_init_xdp_res(struct xsk_socket * xsk,int * xsks_map_fd)782*053f45beSAndroid Build Coastguard Worker static int xsk_init_xdp_res(struct xsk_socket *xsk,
783*053f45beSAndroid Build Coastguard Worker 			    int *xsks_map_fd)
784*053f45beSAndroid Build Coastguard Worker {
785*053f45beSAndroid Build Coastguard Worker 	struct xsk_ctx *ctx = xsk->ctx;
786*053f45beSAndroid Build Coastguard Worker 	int err;
787*053f45beSAndroid Build Coastguard Worker 
788*053f45beSAndroid Build Coastguard Worker 	err = xsk_create_bpf_maps(xsk);
789*053f45beSAndroid Build Coastguard Worker 	if (err)
790*053f45beSAndroid Build Coastguard Worker 		return err;
791*053f45beSAndroid Build Coastguard Worker 
792*053f45beSAndroid Build Coastguard Worker 	err = xsk_load_xdp_prog(xsk);
793*053f45beSAndroid Build Coastguard Worker 	if (err)
794*053f45beSAndroid Build Coastguard Worker 		goto err_load_xdp_prog;
795*053f45beSAndroid Build Coastguard Worker 
796*053f45beSAndroid Build Coastguard Worker 	if (ctx->has_bpf_link)
797*053f45beSAndroid Build Coastguard Worker 		err = xsk_create_bpf_link(xsk);
798*053f45beSAndroid Build Coastguard Worker 	else
799*053f45beSAndroid Build Coastguard Worker 		err = bpf_xdp_attach(xsk->ctx->ifindex, ctx->prog_fd,
800*053f45beSAndroid Build Coastguard Worker 				     xsk->config.xdp_flags, NULL);
801*053f45beSAndroid Build Coastguard Worker 
802*053f45beSAndroid Build Coastguard Worker 	if (err)
803*053f45beSAndroid Build Coastguard Worker 		goto err_attach_xdp_prog;
804*053f45beSAndroid Build Coastguard Worker 
805*053f45beSAndroid Build Coastguard Worker 	if (!xsk->rx)
806*053f45beSAndroid Build Coastguard Worker 		return err;
807*053f45beSAndroid Build Coastguard Worker 
808*053f45beSAndroid Build Coastguard Worker 	err = xsk_set_bpf_maps(xsk);
809*053f45beSAndroid Build Coastguard Worker 	if (err)
810*053f45beSAndroid Build Coastguard Worker 		goto err_set_bpf_maps;
811*053f45beSAndroid Build Coastguard Worker 
812*053f45beSAndroid Build Coastguard Worker 	return err;
813*053f45beSAndroid Build Coastguard Worker 
814*053f45beSAndroid Build Coastguard Worker err_set_bpf_maps:
815*053f45beSAndroid Build Coastguard Worker 	if (ctx->has_bpf_link)
816*053f45beSAndroid Build Coastguard Worker 		close(ctx->link_fd);
817*053f45beSAndroid Build Coastguard Worker 	else
818*053f45beSAndroid Build Coastguard Worker 		bpf_xdp_detach(ctx->ifindex, 0, NULL);
819*053f45beSAndroid Build Coastguard Worker err_attach_xdp_prog:
820*053f45beSAndroid Build Coastguard Worker 	close(ctx->prog_fd);
821*053f45beSAndroid Build Coastguard Worker err_load_xdp_prog:
822*053f45beSAndroid Build Coastguard Worker 	xsk_delete_bpf_maps(xsk);
823*053f45beSAndroid Build Coastguard Worker 	return err;
824*053f45beSAndroid Build Coastguard Worker }
825*053f45beSAndroid Build Coastguard Worker 
xsk_lookup_xdp_res(struct xsk_socket * xsk,int * xsks_map_fd,int prog_id)826*053f45beSAndroid Build Coastguard Worker static int xsk_lookup_xdp_res(struct xsk_socket *xsk, int *xsks_map_fd, int prog_id)
827*053f45beSAndroid Build Coastguard Worker {
828*053f45beSAndroid Build Coastguard Worker 	struct xsk_ctx *ctx = xsk->ctx;
829*053f45beSAndroid Build Coastguard Worker 	int err;
830*053f45beSAndroid Build Coastguard Worker 
831*053f45beSAndroid Build Coastguard Worker 	ctx->prog_fd = bpf_prog_get_fd_by_id(prog_id);
832*053f45beSAndroid Build Coastguard Worker 	if (ctx->prog_fd < 0) {
833*053f45beSAndroid Build Coastguard Worker 		err = -errno;
834*053f45beSAndroid Build Coastguard Worker 		goto err_prog_fd;
835*053f45beSAndroid Build Coastguard Worker 	}
836*053f45beSAndroid Build Coastguard Worker 	err = xsk_lookup_bpf_maps(xsk);
837*053f45beSAndroid Build Coastguard Worker 	if (err)
838*053f45beSAndroid Build Coastguard Worker 		goto err_lookup_maps;
839*053f45beSAndroid Build Coastguard Worker 
840*053f45beSAndroid Build Coastguard Worker 	if (!xsk->rx)
841*053f45beSAndroid Build Coastguard Worker 		return err;
842*053f45beSAndroid Build Coastguard Worker 
843*053f45beSAndroid Build Coastguard Worker 	err = xsk_set_bpf_maps(xsk);
844*053f45beSAndroid Build Coastguard Worker 	if (err)
845*053f45beSAndroid Build Coastguard Worker 		goto err_set_maps;
846*053f45beSAndroid Build Coastguard Worker 
847*053f45beSAndroid Build Coastguard Worker 	return err;
848*053f45beSAndroid Build Coastguard Worker 
849*053f45beSAndroid Build Coastguard Worker err_set_maps:
850*053f45beSAndroid Build Coastguard Worker 	close(ctx->xsks_map_fd);
851*053f45beSAndroid Build Coastguard Worker err_lookup_maps:
852*053f45beSAndroid Build Coastguard Worker 	close(ctx->prog_fd);
853*053f45beSAndroid Build Coastguard Worker err_prog_fd:
854*053f45beSAndroid Build Coastguard Worker 	if (ctx->has_bpf_link)
855*053f45beSAndroid Build Coastguard Worker 		close(ctx->link_fd);
856*053f45beSAndroid Build Coastguard Worker 	return err;
857*053f45beSAndroid Build Coastguard Worker }
858*053f45beSAndroid Build Coastguard Worker 
__xsk_setup_xdp_prog(struct xsk_socket * _xdp,int * xsks_map_fd)859*053f45beSAndroid Build Coastguard Worker static int __xsk_setup_xdp_prog(struct xsk_socket *_xdp, int *xsks_map_fd)
860*053f45beSAndroid Build Coastguard Worker {
861*053f45beSAndroid Build Coastguard Worker 	struct xsk_socket *xsk = _xdp;
862*053f45beSAndroid Build Coastguard Worker 	struct xsk_ctx *ctx = xsk->ctx;
863*053f45beSAndroid Build Coastguard Worker 	__u32 prog_id = 0;
864*053f45beSAndroid Build Coastguard Worker 	int err;
865*053f45beSAndroid Build Coastguard Worker 
866*053f45beSAndroid Build Coastguard Worker 	if (ctx->has_bpf_link)
867*053f45beSAndroid Build Coastguard Worker 		err = xsk_link_lookup(ctx->ifindex, &prog_id, &ctx->link_fd);
868*053f45beSAndroid Build Coastguard Worker 	else
869*053f45beSAndroid Build Coastguard Worker 		err = bpf_xdp_query_id(ctx->ifindex, xsk->config.xdp_flags, &prog_id);
870*053f45beSAndroid Build Coastguard Worker 
871*053f45beSAndroid Build Coastguard Worker 	if (err)
872*053f45beSAndroid Build Coastguard Worker 		return err;
873*053f45beSAndroid Build Coastguard Worker 
874*053f45beSAndroid Build Coastguard Worker 	err = !prog_id ? xsk_init_xdp_res(xsk, xsks_map_fd) :
875*053f45beSAndroid Build Coastguard Worker 			 xsk_lookup_xdp_res(xsk, xsks_map_fd, prog_id);
876*053f45beSAndroid Build Coastguard Worker 
877*053f45beSAndroid Build Coastguard Worker 	if (!err && xsks_map_fd)
878*053f45beSAndroid Build Coastguard Worker 		*xsks_map_fd = ctx->xsks_map_fd;
879*053f45beSAndroid Build Coastguard Worker 
880*053f45beSAndroid Build Coastguard Worker 	return err;
881*053f45beSAndroid Build Coastguard Worker }
882*053f45beSAndroid Build Coastguard Worker 
xsk_setup_xdp_prog_xsk(struct xsk_socket * xsk,int * xsks_map_fd)883*053f45beSAndroid Build Coastguard Worker int xsk_setup_xdp_prog_xsk(struct xsk_socket *xsk, int *xsks_map_fd)
884*053f45beSAndroid Build Coastguard Worker {
885*053f45beSAndroid Build Coastguard Worker 	return __xsk_setup_xdp_prog(xsk, xsks_map_fd);
886*053f45beSAndroid Build Coastguard Worker }
887*053f45beSAndroid Build Coastguard Worker 
xsk_get_ctx(struct xsk_umem * umem,int ifindex,__u32 queue_id)888*053f45beSAndroid Build Coastguard Worker static struct xsk_ctx *xsk_get_ctx(struct xsk_umem *umem, int ifindex,
889*053f45beSAndroid Build Coastguard Worker 				   __u32 queue_id)
890*053f45beSAndroid Build Coastguard Worker {
891*053f45beSAndroid Build Coastguard Worker 	struct xsk_ctx *ctx;
892*053f45beSAndroid Build Coastguard Worker 
893*053f45beSAndroid Build Coastguard Worker 	if (list_empty(&umem->ctx_list))
894*053f45beSAndroid Build Coastguard Worker 		return NULL;
895*053f45beSAndroid Build Coastguard Worker 
896*053f45beSAndroid Build Coastguard Worker 	list_for_each_entry(ctx, &umem->ctx_list, list) {
897*053f45beSAndroid Build Coastguard Worker 		if (ctx->ifindex == ifindex && ctx->queue_id == queue_id) {
898*053f45beSAndroid Build Coastguard Worker 			ctx->refcount++;
899*053f45beSAndroid Build Coastguard Worker 			return ctx;
900*053f45beSAndroid Build Coastguard Worker 		}
901*053f45beSAndroid Build Coastguard Worker 	}
902*053f45beSAndroid Build Coastguard Worker 
903*053f45beSAndroid Build Coastguard Worker 	return NULL;
904*053f45beSAndroid Build Coastguard Worker }
905*053f45beSAndroid Build Coastguard Worker 
xsk_put_ctx(struct xsk_ctx * ctx,bool unmap)906*053f45beSAndroid Build Coastguard Worker static void xsk_put_ctx(struct xsk_ctx *ctx, bool unmap)
907*053f45beSAndroid Build Coastguard Worker {
908*053f45beSAndroid Build Coastguard Worker 	struct xsk_umem *umem = ctx->umem;
909*053f45beSAndroid Build Coastguard Worker 	struct xdp_mmap_offsets off;
910*053f45beSAndroid Build Coastguard Worker 	int err;
911*053f45beSAndroid Build Coastguard Worker 
912*053f45beSAndroid Build Coastguard Worker 	if (--ctx->refcount)
913*053f45beSAndroid Build Coastguard Worker 		return;
914*053f45beSAndroid Build Coastguard Worker 
915*053f45beSAndroid Build Coastguard Worker 	if (!unmap)
916*053f45beSAndroid Build Coastguard Worker 		goto out_free;
917*053f45beSAndroid Build Coastguard Worker 
918*053f45beSAndroid Build Coastguard Worker 	err = xsk_get_mmap_offsets(umem->fd, &off);
919*053f45beSAndroid Build Coastguard Worker 	if (err)
920*053f45beSAndroid Build Coastguard Worker 		goto out_free;
921*053f45beSAndroid Build Coastguard Worker 
922*053f45beSAndroid Build Coastguard Worker 	munmap(ctx->fill->ring - off.fr.desc, off.fr.desc + umem->config.fill_size *
923*053f45beSAndroid Build Coastguard Worker 	       sizeof(__u64));
924*053f45beSAndroid Build Coastguard Worker 	munmap(ctx->comp->ring - off.cr.desc, off.cr.desc + umem->config.comp_size *
925*053f45beSAndroid Build Coastguard Worker 	       sizeof(__u64));
926*053f45beSAndroid Build Coastguard Worker 
927*053f45beSAndroid Build Coastguard Worker out_free:
928*053f45beSAndroid Build Coastguard Worker 	list_del(&ctx->list);
929*053f45beSAndroid Build Coastguard Worker 	free(ctx);
930*053f45beSAndroid Build Coastguard Worker }
931*053f45beSAndroid Build Coastguard Worker 
xsk_create_ctx(struct xsk_socket * xsk,struct xsk_umem * umem,int ifindex,const char * ifname,__u32 queue_id,struct xsk_ring_prod * fill,struct xsk_ring_cons * comp)932*053f45beSAndroid Build Coastguard Worker static struct xsk_ctx *xsk_create_ctx(struct xsk_socket *xsk,
933*053f45beSAndroid Build Coastguard Worker 				      struct xsk_umem *umem, int ifindex,
934*053f45beSAndroid Build Coastguard Worker 				      const char *ifname, __u32 queue_id,
935*053f45beSAndroid Build Coastguard Worker 				      struct xsk_ring_prod *fill,
936*053f45beSAndroid Build Coastguard Worker 				      struct xsk_ring_cons *comp)
937*053f45beSAndroid Build Coastguard Worker {
938*053f45beSAndroid Build Coastguard Worker 	struct xsk_ctx *ctx;
939*053f45beSAndroid Build Coastguard Worker 	int err;
940*053f45beSAndroid Build Coastguard Worker 
941*053f45beSAndroid Build Coastguard Worker 	ctx = calloc(1, sizeof(*ctx));
942*053f45beSAndroid Build Coastguard Worker 	if (!ctx)
943*053f45beSAndroid Build Coastguard Worker 		return NULL;
944*053f45beSAndroid Build Coastguard Worker 
945*053f45beSAndroid Build Coastguard Worker 	if (!umem->fill_save) {
946*053f45beSAndroid Build Coastguard Worker 		err = xsk_create_umem_rings(umem, xsk->fd, fill, comp);
947*053f45beSAndroid Build Coastguard Worker 		if (err) {
948*053f45beSAndroid Build Coastguard Worker 			free(ctx);
949*053f45beSAndroid Build Coastguard Worker 			return NULL;
950*053f45beSAndroid Build Coastguard Worker 		}
951*053f45beSAndroid Build Coastguard Worker 	} else if (umem->fill_save != fill || umem->comp_save != comp) {
952*053f45beSAndroid Build Coastguard Worker 		/* Copy over rings to new structs. */
953*053f45beSAndroid Build Coastguard Worker 		memcpy(fill, umem->fill_save, sizeof(*fill));
954*053f45beSAndroid Build Coastguard Worker 		memcpy(comp, umem->comp_save, sizeof(*comp));
955*053f45beSAndroid Build Coastguard Worker 	}
956*053f45beSAndroid Build Coastguard Worker 
957*053f45beSAndroid Build Coastguard Worker 	ctx->ifindex = ifindex;
958*053f45beSAndroid Build Coastguard Worker 	ctx->refcount = 1;
959*053f45beSAndroid Build Coastguard Worker 	ctx->umem = umem;
960*053f45beSAndroid Build Coastguard Worker 	ctx->queue_id = queue_id;
961*053f45beSAndroid Build Coastguard Worker 	libbpf_strlcpy(ctx->ifname, ifname, IFNAMSIZ);
962*053f45beSAndroid Build Coastguard Worker 
963*053f45beSAndroid Build Coastguard Worker 	ctx->fill = fill;
964*053f45beSAndroid Build Coastguard Worker 	ctx->comp = comp;
965*053f45beSAndroid Build Coastguard Worker 	list_add(&ctx->list, &umem->ctx_list);
966*053f45beSAndroid Build Coastguard Worker 	ctx->has_bpf_link = xsk_probe_bpf_link();
967*053f45beSAndroid Build Coastguard Worker 	return ctx;
968*053f45beSAndroid Build Coastguard Worker }
969*053f45beSAndroid Build Coastguard Worker 
xsk_destroy_xsk_struct(struct xsk_socket * xsk)970*053f45beSAndroid Build Coastguard Worker static void xsk_destroy_xsk_struct(struct xsk_socket *xsk)
971*053f45beSAndroid Build Coastguard Worker {
972*053f45beSAndroid Build Coastguard Worker 	free(xsk->ctx);
973*053f45beSAndroid Build Coastguard Worker 	free(xsk);
974*053f45beSAndroid Build Coastguard Worker }
975*053f45beSAndroid Build Coastguard Worker 
xsk_socket__update_xskmap(struct xsk_socket * xsk,int fd)976*053f45beSAndroid Build Coastguard Worker int xsk_socket__update_xskmap(struct xsk_socket *xsk, int fd)
977*053f45beSAndroid Build Coastguard Worker {
978*053f45beSAndroid Build Coastguard Worker 	xsk->ctx->xsks_map_fd = fd;
979*053f45beSAndroid Build Coastguard Worker 	return xsk_set_bpf_maps(xsk);
980*053f45beSAndroid Build Coastguard Worker }
981*053f45beSAndroid Build Coastguard Worker 
xsk_setup_xdp_prog(int ifindex,int * xsks_map_fd)982*053f45beSAndroid Build Coastguard Worker int xsk_setup_xdp_prog(int ifindex, int *xsks_map_fd)
983*053f45beSAndroid Build Coastguard Worker {
984*053f45beSAndroid Build Coastguard Worker 	struct xsk_socket *xsk;
985*053f45beSAndroid Build Coastguard Worker 	int res;
986*053f45beSAndroid Build Coastguard Worker 
987*053f45beSAndroid Build Coastguard Worker 	xsk = calloc(1, sizeof(*xsk));
988*053f45beSAndroid Build Coastguard Worker 	if (!xsk)
989*053f45beSAndroid Build Coastguard Worker 		return -ENOMEM;
990*053f45beSAndroid Build Coastguard Worker 
991*053f45beSAndroid Build Coastguard Worker 	res = xsk_create_xsk_struct(ifindex, xsk);
992*053f45beSAndroid Build Coastguard Worker 	if (res) {
993*053f45beSAndroid Build Coastguard Worker 		free(xsk);
994*053f45beSAndroid Build Coastguard Worker 		return -EINVAL;
995*053f45beSAndroid Build Coastguard Worker 	}
996*053f45beSAndroid Build Coastguard Worker 
997*053f45beSAndroid Build Coastguard Worker 	res = __xsk_setup_xdp_prog(xsk, xsks_map_fd);
998*053f45beSAndroid Build Coastguard Worker 
999*053f45beSAndroid Build Coastguard Worker 	xsk_destroy_xsk_struct(xsk);
1000*053f45beSAndroid Build Coastguard Worker 
1001*053f45beSAndroid Build Coastguard Worker 	return res;
1002*053f45beSAndroid Build Coastguard Worker }
1003*053f45beSAndroid Build Coastguard Worker 
xsk_socket__create_shared(struct xsk_socket ** xsk_ptr,const char * ifname,__u32 queue_id,struct xsk_umem * umem,struct xsk_ring_cons * rx,struct xsk_ring_prod * tx,struct xsk_ring_prod * fill,struct xsk_ring_cons * comp,const struct xsk_socket_config * usr_config)1004*053f45beSAndroid Build Coastguard Worker int xsk_socket__create_shared(struct xsk_socket **xsk_ptr,
1005*053f45beSAndroid Build Coastguard Worker 			      const char *ifname,
1006*053f45beSAndroid Build Coastguard Worker 			      __u32 queue_id, struct xsk_umem *umem,
1007*053f45beSAndroid Build Coastguard Worker 			      struct xsk_ring_cons *rx,
1008*053f45beSAndroid Build Coastguard Worker 			      struct xsk_ring_prod *tx,
1009*053f45beSAndroid Build Coastguard Worker 			      struct xsk_ring_prod *fill,
1010*053f45beSAndroid Build Coastguard Worker 			      struct xsk_ring_cons *comp,
1011*053f45beSAndroid Build Coastguard Worker 			      const struct xsk_socket_config *usr_config)
1012*053f45beSAndroid Build Coastguard Worker {
1013*053f45beSAndroid Build Coastguard Worker 	bool unmap, rx_setup_done = false, tx_setup_done = false;
1014*053f45beSAndroid Build Coastguard Worker 	void *rx_map = NULL, *tx_map = NULL;
1015*053f45beSAndroid Build Coastguard Worker 	struct sockaddr_xdp sxdp = {};
1016*053f45beSAndroid Build Coastguard Worker 	struct xdp_mmap_offsets off;
1017*053f45beSAndroid Build Coastguard Worker 	struct xsk_socket *xsk;
1018*053f45beSAndroid Build Coastguard Worker 	struct xsk_ctx *ctx;
1019*053f45beSAndroid Build Coastguard Worker 	int err, ifindex;
1020*053f45beSAndroid Build Coastguard Worker 
1021*053f45beSAndroid Build Coastguard Worker 	if (!umem || !xsk_ptr || !(rx || tx))
1022*053f45beSAndroid Build Coastguard Worker 		return -EFAULT;
1023*053f45beSAndroid Build Coastguard Worker 
1024*053f45beSAndroid Build Coastguard Worker 	unmap = umem->fill_save != fill;
1025*053f45beSAndroid Build Coastguard Worker 
1026*053f45beSAndroid Build Coastguard Worker 	xsk = calloc(1, sizeof(*xsk));
1027*053f45beSAndroid Build Coastguard Worker 	if (!xsk)
1028*053f45beSAndroid Build Coastguard Worker 		return -ENOMEM;
1029*053f45beSAndroid Build Coastguard Worker 
1030*053f45beSAndroid Build Coastguard Worker 	err = xsk_set_xdp_socket_config(&xsk->config, usr_config);
1031*053f45beSAndroid Build Coastguard Worker 	if (err)
1032*053f45beSAndroid Build Coastguard Worker 		goto out_xsk_alloc;
1033*053f45beSAndroid Build Coastguard Worker 
1034*053f45beSAndroid Build Coastguard Worker 	xsk->outstanding_tx = 0;
1035*053f45beSAndroid Build Coastguard Worker 	ifindex = if_nametoindex(ifname);
1036*053f45beSAndroid Build Coastguard Worker 	if (!ifindex) {
1037*053f45beSAndroid Build Coastguard Worker 		err = -errno;
1038*053f45beSAndroid Build Coastguard Worker 		goto out_xsk_alloc;
1039*053f45beSAndroid Build Coastguard Worker 	}
1040*053f45beSAndroid Build Coastguard Worker 
1041*053f45beSAndroid Build Coastguard Worker 	if (umem->refcount++ > 0) {
1042*053f45beSAndroid Build Coastguard Worker 		xsk->fd = socket(AF_XDP, SOCK_RAW | SOCK_CLOEXEC, 0);
1043*053f45beSAndroid Build Coastguard Worker 		if (xsk->fd < 0) {
1044*053f45beSAndroid Build Coastguard Worker 			err = -errno;
1045*053f45beSAndroid Build Coastguard Worker 			goto out_xsk_alloc;
1046*053f45beSAndroid Build Coastguard Worker 		}
1047*053f45beSAndroid Build Coastguard Worker 	} else {
1048*053f45beSAndroid Build Coastguard Worker 		xsk->fd = umem->fd;
1049*053f45beSAndroid Build Coastguard Worker 		rx_setup_done = umem->rx_ring_setup_done;
1050*053f45beSAndroid Build Coastguard Worker 		tx_setup_done = umem->tx_ring_setup_done;
1051*053f45beSAndroid Build Coastguard Worker 	}
1052*053f45beSAndroid Build Coastguard Worker 
1053*053f45beSAndroid Build Coastguard Worker 	ctx = xsk_get_ctx(umem, ifindex, queue_id);
1054*053f45beSAndroid Build Coastguard Worker 	if (!ctx) {
1055*053f45beSAndroid Build Coastguard Worker 		if (!fill || !comp) {
1056*053f45beSAndroid Build Coastguard Worker 			err = -EFAULT;
1057*053f45beSAndroid Build Coastguard Worker 			goto out_socket;
1058*053f45beSAndroid Build Coastguard Worker 		}
1059*053f45beSAndroid Build Coastguard Worker 
1060*053f45beSAndroid Build Coastguard Worker 		ctx = xsk_create_ctx(xsk, umem, ifindex, ifname, queue_id,
1061*053f45beSAndroid Build Coastguard Worker 				     fill, comp);
1062*053f45beSAndroid Build Coastguard Worker 		if (!ctx) {
1063*053f45beSAndroid Build Coastguard Worker 			err = -ENOMEM;
1064*053f45beSAndroid Build Coastguard Worker 			goto out_socket;
1065*053f45beSAndroid Build Coastguard Worker 		}
1066*053f45beSAndroid Build Coastguard Worker 	}
1067*053f45beSAndroid Build Coastguard Worker 	xsk->ctx = ctx;
1068*053f45beSAndroid Build Coastguard Worker 
1069*053f45beSAndroid Build Coastguard Worker 	if (rx && !rx_setup_done) {
1070*053f45beSAndroid Build Coastguard Worker 		err = setsockopt(xsk->fd, SOL_XDP, XDP_RX_RING,
1071*053f45beSAndroid Build Coastguard Worker 				 &xsk->config.rx_size,
1072*053f45beSAndroid Build Coastguard Worker 				 sizeof(xsk->config.rx_size));
1073*053f45beSAndroid Build Coastguard Worker 		if (err) {
1074*053f45beSAndroid Build Coastguard Worker 			err = -errno;
1075*053f45beSAndroid Build Coastguard Worker 			goto out_put_ctx;
1076*053f45beSAndroid Build Coastguard Worker 		}
1077*053f45beSAndroid Build Coastguard Worker 		if (xsk->fd == umem->fd)
1078*053f45beSAndroid Build Coastguard Worker 			umem->rx_ring_setup_done = true;
1079*053f45beSAndroid Build Coastguard Worker 	}
1080*053f45beSAndroid Build Coastguard Worker 	if (tx && !tx_setup_done) {
1081*053f45beSAndroid Build Coastguard Worker 		err = setsockopt(xsk->fd, SOL_XDP, XDP_TX_RING,
1082*053f45beSAndroid Build Coastguard Worker 				 &xsk->config.tx_size,
1083*053f45beSAndroid Build Coastguard Worker 				 sizeof(xsk->config.tx_size));
1084*053f45beSAndroid Build Coastguard Worker 		if (err) {
1085*053f45beSAndroid Build Coastguard Worker 			err = -errno;
1086*053f45beSAndroid Build Coastguard Worker 			goto out_put_ctx;
1087*053f45beSAndroid Build Coastguard Worker 		}
1088*053f45beSAndroid Build Coastguard Worker 		if (xsk->fd == umem->fd)
1089*053f45beSAndroid Build Coastguard Worker 			umem->tx_ring_setup_done = true;
1090*053f45beSAndroid Build Coastguard Worker 	}
1091*053f45beSAndroid Build Coastguard Worker 
1092*053f45beSAndroid Build Coastguard Worker 	err = xsk_get_mmap_offsets(xsk->fd, &off);
1093*053f45beSAndroid Build Coastguard Worker 	if (err) {
1094*053f45beSAndroid Build Coastguard Worker 		err = -errno;
1095*053f45beSAndroid Build Coastguard Worker 		goto out_put_ctx;
1096*053f45beSAndroid Build Coastguard Worker 	}
1097*053f45beSAndroid Build Coastguard Worker 
1098*053f45beSAndroid Build Coastguard Worker 	if (rx) {
1099*053f45beSAndroid Build Coastguard Worker 		rx_map = mmap(NULL, off.rx.desc +
1100*053f45beSAndroid Build Coastguard Worker 			      xsk->config.rx_size * sizeof(struct xdp_desc),
1101*053f45beSAndroid Build Coastguard Worker 			      PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
1102*053f45beSAndroid Build Coastguard Worker 			      xsk->fd, XDP_PGOFF_RX_RING);
1103*053f45beSAndroid Build Coastguard Worker 		if (rx_map == MAP_FAILED) {
1104*053f45beSAndroid Build Coastguard Worker 			err = -errno;
1105*053f45beSAndroid Build Coastguard Worker 			goto out_put_ctx;
1106*053f45beSAndroid Build Coastguard Worker 		}
1107*053f45beSAndroid Build Coastguard Worker 
1108*053f45beSAndroid Build Coastguard Worker 		rx->mask = xsk->config.rx_size - 1;
1109*053f45beSAndroid Build Coastguard Worker 		rx->size = xsk->config.rx_size;
1110*053f45beSAndroid Build Coastguard Worker 		rx->producer = rx_map + off.rx.producer;
1111*053f45beSAndroid Build Coastguard Worker 		rx->consumer = rx_map + off.rx.consumer;
1112*053f45beSAndroid Build Coastguard Worker 		rx->flags = rx_map + off.rx.flags;
1113*053f45beSAndroid Build Coastguard Worker 		rx->ring = rx_map + off.rx.desc;
1114*053f45beSAndroid Build Coastguard Worker 		rx->cached_prod = *rx->producer;
1115*053f45beSAndroid Build Coastguard Worker 		rx->cached_cons = *rx->consumer;
1116*053f45beSAndroid Build Coastguard Worker 	}
1117*053f45beSAndroid Build Coastguard Worker 	xsk->rx = rx;
1118*053f45beSAndroid Build Coastguard Worker 
1119*053f45beSAndroid Build Coastguard Worker 	if (tx) {
1120*053f45beSAndroid Build Coastguard Worker 		tx_map = mmap(NULL, off.tx.desc +
1121*053f45beSAndroid Build Coastguard Worker 			      xsk->config.tx_size * sizeof(struct xdp_desc),
1122*053f45beSAndroid Build Coastguard Worker 			      PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
1123*053f45beSAndroid Build Coastguard Worker 			      xsk->fd, XDP_PGOFF_TX_RING);
1124*053f45beSAndroid Build Coastguard Worker 		if (tx_map == MAP_FAILED) {
1125*053f45beSAndroid Build Coastguard Worker 			err = -errno;
1126*053f45beSAndroid Build Coastguard Worker 			goto out_mmap_rx;
1127*053f45beSAndroid Build Coastguard Worker 		}
1128*053f45beSAndroid Build Coastguard Worker 
1129*053f45beSAndroid Build Coastguard Worker 		tx->mask = xsk->config.tx_size - 1;
1130*053f45beSAndroid Build Coastguard Worker 		tx->size = xsk->config.tx_size;
1131*053f45beSAndroid Build Coastguard Worker 		tx->producer = tx_map + off.tx.producer;
1132*053f45beSAndroid Build Coastguard Worker 		tx->consumer = tx_map + off.tx.consumer;
1133*053f45beSAndroid Build Coastguard Worker 		tx->flags = tx_map + off.tx.flags;
1134*053f45beSAndroid Build Coastguard Worker 		tx->ring = tx_map + off.tx.desc;
1135*053f45beSAndroid Build Coastguard Worker 		tx->cached_prod = *tx->producer;
1136*053f45beSAndroid Build Coastguard Worker 		/* cached_cons is r->size bigger than the real consumer pointer
1137*053f45beSAndroid Build Coastguard Worker 		 * See xsk_prod_nb_free
1138*053f45beSAndroid Build Coastguard Worker 		 */
1139*053f45beSAndroid Build Coastguard Worker 		tx->cached_cons = *tx->consumer + xsk->config.tx_size;
1140*053f45beSAndroid Build Coastguard Worker 	}
1141*053f45beSAndroid Build Coastguard Worker 	xsk->tx = tx;
1142*053f45beSAndroid Build Coastguard Worker 
1143*053f45beSAndroid Build Coastguard Worker 	sxdp.sxdp_family = PF_XDP;
1144*053f45beSAndroid Build Coastguard Worker 	sxdp.sxdp_ifindex = ctx->ifindex;
1145*053f45beSAndroid Build Coastguard Worker 	sxdp.sxdp_queue_id = ctx->queue_id;
1146*053f45beSAndroid Build Coastguard Worker 	if (umem->refcount > 1) {
1147*053f45beSAndroid Build Coastguard Worker 		sxdp.sxdp_flags |= XDP_SHARED_UMEM;
1148*053f45beSAndroid Build Coastguard Worker 		sxdp.sxdp_shared_umem_fd = umem->fd;
1149*053f45beSAndroid Build Coastguard Worker 	} else {
1150*053f45beSAndroid Build Coastguard Worker 		sxdp.sxdp_flags = xsk->config.bind_flags;
1151*053f45beSAndroid Build Coastguard Worker 	}
1152*053f45beSAndroid Build Coastguard Worker 
1153*053f45beSAndroid Build Coastguard Worker 	err = bind(xsk->fd, (struct sockaddr *)&sxdp, sizeof(sxdp));
1154*053f45beSAndroid Build Coastguard Worker 	if (err) {
1155*053f45beSAndroid Build Coastguard Worker 		err = -errno;
1156*053f45beSAndroid Build Coastguard Worker 		goto out_mmap_tx;
1157*053f45beSAndroid Build Coastguard Worker 	}
1158*053f45beSAndroid Build Coastguard Worker 
1159*053f45beSAndroid Build Coastguard Worker 	if (!(xsk->config.libbpf_flags & XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD)) {
1160*053f45beSAndroid Build Coastguard Worker 		err = __xsk_setup_xdp_prog(xsk, NULL);
1161*053f45beSAndroid Build Coastguard Worker 		if (err)
1162*053f45beSAndroid Build Coastguard Worker 			goto out_mmap_tx;
1163*053f45beSAndroid Build Coastguard Worker 	}
1164*053f45beSAndroid Build Coastguard Worker 
1165*053f45beSAndroid Build Coastguard Worker 	*xsk_ptr = xsk;
1166*053f45beSAndroid Build Coastguard Worker 	umem->fill_save = NULL;
1167*053f45beSAndroid Build Coastguard Worker 	umem->comp_save = NULL;
1168*053f45beSAndroid Build Coastguard Worker 	return 0;
1169*053f45beSAndroid Build Coastguard Worker 
1170*053f45beSAndroid Build Coastguard Worker out_mmap_tx:
1171*053f45beSAndroid Build Coastguard Worker 	if (tx)
1172*053f45beSAndroid Build Coastguard Worker 		munmap(tx_map, off.tx.desc +
1173*053f45beSAndroid Build Coastguard Worker 		       xsk->config.tx_size * sizeof(struct xdp_desc));
1174*053f45beSAndroid Build Coastguard Worker out_mmap_rx:
1175*053f45beSAndroid Build Coastguard Worker 	if (rx)
1176*053f45beSAndroid Build Coastguard Worker 		munmap(rx_map, off.rx.desc +
1177*053f45beSAndroid Build Coastguard Worker 		       xsk->config.rx_size * sizeof(struct xdp_desc));
1178*053f45beSAndroid Build Coastguard Worker out_put_ctx:
1179*053f45beSAndroid Build Coastguard Worker 	xsk_put_ctx(ctx, unmap);
1180*053f45beSAndroid Build Coastguard Worker out_socket:
1181*053f45beSAndroid Build Coastguard Worker 	if (--umem->refcount)
1182*053f45beSAndroid Build Coastguard Worker 		close(xsk->fd);
1183*053f45beSAndroid Build Coastguard Worker out_xsk_alloc:
1184*053f45beSAndroid Build Coastguard Worker 	free(xsk);
1185*053f45beSAndroid Build Coastguard Worker 	return err;
1186*053f45beSAndroid Build Coastguard Worker }
1187*053f45beSAndroid Build Coastguard Worker 
xsk_socket__create(struct xsk_socket ** xsk_ptr,const char * ifname,__u32 queue_id,struct xsk_umem * umem,struct xsk_ring_cons * rx,struct xsk_ring_prod * tx,const struct xsk_socket_config * usr_config)1188*053f45beSAndroid Build Coastguard Worker int xsk_socket__create(struct xsk_socket **xsk_ptr, const char *ifname,
1189*053f45beSAndroid Build Coastguard Worker 		       __u32 queue_id, struct xsk_umem *umem,
1190*053f45beSAndroid Build Coastguard Worker 		       struct xsk_ring_cons *rx, struct xsk_ring_prod *tx,
1191*053f45beSAndroid Build Coastguard Worker 		       const struct xsk_socket_config *usr_config)
1192*053f45beSAndroid Build Coastguard Worker {
1193*053f45beSAndroid Build Coastguard Worker 	if (!umem)
1194*053f45beSAndroid Build Coastguard Worker 		return -EFAULT;
1195*053f45beSAndroid Build Coastguard Worker 
1196*053f45beSAndroid Build Coastguard Worker 	return xsk_socket__create_shared(xsk_ptr, ifname, queue_id, umem,
1197*053f45beSAndroid Build Coastguard Worker 					 rx, tx, umem->fill_save,
1198*053f45beSAndroid Build Coastguard Worker 					 umem->comp_save, usr_config);
1199*053f45beSAndroid Build Coastguard Worker }
1200*053f45beSAndroid Build Coastguard Worker 
xsk_umem__delete(struct xsk_umem * umem)1201*053f45beSAndroid Build Coastguard Worker int xsk_umem__delete(struct xsk_umem *umem)
1202*053f45beSAndroid Build Coastguard Worker {
1203*053f45beSAndroid Build Coastguard Worker 	struct xdp_mmap_offsets off;
1204*053f45beSAndroid Build Coastguard Worker 	int err;
1205*053f45beSAndroid Build Coastguard Worker 
1206*053f45beSAndroid Build Coastguard Worker 	if (!umem)
1207*053f45beSAndroid Build Coastguard Worker 		return 0;
1208*053f45beSAndroid Build Coastguard Worker 
1209*053f45beSAndroid Build Coastguard Worker 	if (umem->refcount)
1210*053f45beSAndroid Build Coastguard Worker 		return -EBUSY;
1211*053f45beSAndroid Build Coastguard Worker 
1212*053f45beSAndroid Build Coastguard Worker 	err = xsk_get_mmap_offsets(umem->fd, &off);
1213*053f45beSAndroid Build Coastguard Worker 	if (!err && umem->fill_save && umem->comp_save) {
1214*053f45beSAndroid Build Coastguard Worker 		munmap(umem->fill_save->ring - off.fr.desc,
1215*053f45beSAndroid Build Coastguard Worker 		       off.fr.desc + umem->config.fill_size * sizeof(__u64));
1216*053f45beSAndroid Build Coastguard Worker 		munmap(umem->comp_save->ring - off.cr.desc,
1217*053f45beSAndroid Build Coastguard Worker 		       off.cr.desc + umem->config.comp_size * sizeof(__u64));
1218*053f45beSAndroid Build Coastguard Worker 	}
1219*053f45beSAndroid Build Coastguard Worker 
1220*053f45beSAndroid Build Coastguard Worker 	close(umem->fd);
1221*053f45beSAndroid Build Coastguard Worker 	free(umem);
1222*053f45beSAndroid Build Coastguard Worker 
1223*053f45beSAndroid Build Coastguard Worker 	return 0;
1224*053f45beSAndroid Build Coastguard Worker }
1225*053f45beSAndroid Build Coastguard Worker 
xsk_socket__delete(struct xsk_socket * xsk)1226*053f45beSAndroid Build Coastguard Worker void xsk_socket__delete(struct xsk_socket *xsk)
1227*053f45beSAndroid Build Coastguard Worker {
1228*053f45beSAndroid Build Coastguard Worker 	size_t desc_sz = sizeof(struct xdp_desc);
1229*053f45beSAndroid Build Coastguard Worker 	struct xdp_mmap_offsets off;
1230*053f45beSAndroid Build Coastguard Worker 	struct xsk_umem *umem;
1231*053f45beSAndroid Build Coastguard Worker 	struct xsk_ctx *ctx;
1232*053f45beSAndroid Build Coastguard Worker 	int err;
1233*053f45beSAndroid Build Coastguard Worker 
1234*053f45beSAndroid Build Coastguard Worker 	if (!xsk)
1235*053f45beSAndroid Build Coastguard Worker 		return;
1236*053f45beSAndroid Build Coastguard Worker 
1237*053f45beSAndroid Build Coastguard Worker 	ctx = xsk->ctx;
1238*053f45beSAndroid Build Coastguard Worker 	umem = ctx->umem;
1239*053f45beSAndroid Build Coastguard Worker 
1240*053f45beSAndroid Build Coastguard Worker 	if (ctx->refcount == 1) {
1241*053f45beSAndroid Build Coastguard Worker 		xsk_delete_bpf_maps(xsk);
1242*053f45beSAndroid Build Coastguard Worker 		close(ctx->prog_fd);
1243*053f45beSAndroid Build Coastguard Worker 		if (ctx->has_bpf_link)
1244*053f45beSAndroid Build Coastguard Worker 			close(ctx->link_fd);
1245*053f45beSAndroid Build Coastguard Worker 	}
1246*053f45beSAndroid Build Coastguard Worker 
1247*053f45beSAndroid Build Coastguard Worker 	xsk_put_ctx(ctx, true);
1248*053f45beSAndroid Build Coastguard Worker 
1249*053f45beSAndroid Build Coastguard Worker 	err = xsk_get_mmap_offsets(xsk->fd, &off);
1250*053f45beSAndroid Build Coastguard Worker 	if (!err) {
1251*053f45beSAndroid Build Coastguard Worker 		if (xsk->rx) {
1252*053f45beSAndroid Build Coastguard Worker 			munmap(xsk->rx->ring - off.rx.desc,
1253*053f45beSAndroid Build Coastguard Worker 			       off.rx.desc + xsk->config.rx_size * desc_sz);
1254*053f45beSAndroid Build Coastguard Worker 		}
1255*053f45beSAndroid Build Coastguard Worker 		if (xsk->tx) {
1256*053f45beSAndroid Build Coastguard Worker 			munmap(xsk->tx->ring - off.tx.desc,
1257*053f45beSAndroid Build Coastguard Worker 			       off.tx.desc + xsk->config.tx_size * desc_sz);
1258*053f45beSAndroid Build Coastguard Worker 		}
1259*053f45beSAndroid Build Coastguard Worker 	}
1260*053f45beSAndroid Build Coastguard Worker 
1261*053f45beSAndroid Build Coastguard Worker 	umem->refcount--;
1262*053f45beSAndroid Build Coastguard Worker 	/* Do not close an fd that also has an associated umem connected
1263*053f45beSAndroid Build Coastguard Worker 	 * to it.
1264*053f45beSAndroid Build Coastguard Worker 	 */
1265*053f45beSAndroid Build Coastguard Worker 	if (xsk->fd != umem->fd)
1266*053f45beSAndroid Build Coastguard Worker 		close(xsk->fd);
1267*053f45beSAndroid Build Coastguard Worker 	free(xsk);
1268*053f45beSAndroid Build Coastguard Worker }
1269