1*053f45beSAndroid Build Coastguard Worker // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2*053f45beSAndroid Build Coastguard Worker
3*053f45beSAndroid Build Coastguard Worker /*
4*053f45beSAndroid Build Coastguard Worker * AF_XDP user-space access library.
5*053f45beSAndroid Build Coastguard Worker *
6*053f45beSAndroid Build Coastguard Worker * Copyright(c) 2018 - 2019 Intel Corporation.
7*053f45beSAndroid Build Coastguard Worker *
8*053f45beSAndroid Build Coastguard Worker * Author(s): Magnus Karlsson <[email protected]>
9*053f45beSAndroid Build Coastguard Worker */
10*053f45beSAndroid Build Coastguard Worker
11*053f45beSAndroid Build Coastguard Worker #include <errno.h>
12*053f45beSAndroid Build Coastguard Worker #include <stdlib.h>
13*053f45beSAndroid Build Coastguard Worker #include <string.h>
14*053f45beSAndroid Build Coastguard Worker #include <unistd.h>
15*053f45beSAndroid Build Coastguard Worker #include <arpa/inet.h>
16*053f45beSAndroid Build Coastguard Worker #include <asm/barrier.h>
17*053f45beSAndroid Build Coastguard Worker #include <linux/compiler.h>
18*053f45beSAndroid Build Coastguard Worker #include <linux/ethtool.h>
19*053f45beSAndroid Build Coastguard Worker #include <linux/filter.h>
20*053f45beSAndroid Build Coastguard Worker #include <linux/if_ether.h>
21*053f45beSAndroid Build Coastguard Worker #include <linux/if_packet.h>
22*053f45beSAndroid Build Coastguard Worker #include <linux/if_xdp.h>
23*053f45beSAndroid Build Coastguard Worker #include <linux/kernel.h>
24*053f45beSAndroid Build Coastguard Worker #include <linux/list.h>
25*053f45beSAndroid Build Coastguard Worker #include <linux/sockios.h>
26*053f45beSAndroid Build Coastguard Worker #include <net/if.h>
27*053f45beSAndroid Build Coastguard Worker #include <sys/ioctl.h>
28*053f45beSAndroid Build Coastguard Worker #include <sys/mman.h>
29*053f45beSAndroid Build Coastguard Worker #include <sys/socket.h>
30*053f45beSAndroid Build Coastguard Worker #include <sys/types.h>
31*053f45beSAndroid Build Coastguard Worker #include <linux/if_link.h>
32*053f45beSAndroid Build Coastguard Worker
33*053f45beSAndroid Build Coastguard Worker #include <bpf/bpf.h>
34*053f45beSAndroid Build Coastguard Worker #include <bpf/libbpf.h>
35*053f45beSAndroid Build Coastguard Worker #include "xsk.h"
36*053f45beSAndroid Build Coastguard Worker
37*053f45beSAndroid Build Coastguard Worker #ifndef SOL_XDP
38*053f45beSAndroid Build Coastguard Worker #define SOL_XDP 283
39*053f45beSAndroid Build Coastguard Worker #endif
40*053f45beSAndroid Build Coastguard Worker
41*053f45beSAndroid Build Coastguard Worker #ifndef AF_XDP
42*053f45beSAndroid Build Coastguard Worker #define AF_XDP 44
43*053f45beSAndroid Build Coastguard Worker #endif
44*053f45beSAndroid Build Coastguard Worker
45*053f45beSAndroid Build Coastguard Worker #ifndef PF_XDP
46*053f45beSAndroid Build Coastguard Worker #define PF_XDP AF_XDP
47*053f45beSAndroid Build Coastguard Worker #endif
48*053f45beSAndroid Build Coastguard Worker
49*053f45beSAndroid Build Coastguard Worker #define pr_warn(fmt, ...) fprintf(stderr, fmt, ##__VA_ARGS__)
50*053f45beSAndroid Build Coastguard Worker
51*053f45beSAndroid Build Coastguard Worker enum xsk_prog {
52*053f45beSAndroid Build Coastguard Worker XSK_PROG_FALLBACK,
53*053f45beSAndroid Build Coastguard Worker XSK_PROG_REDIRECT_FLAGS,
54*053f45beSAndroid Build Coastguard Worker };
55*053f45beSAndroid Build Coastguard Worker
56*053f45beSAndroid Build Coastguard Worker struct xsk_umem {
57*053f45beSAndroid Build Coastguard Worker struct xsk_ring_prod *fill_save;
58*053f45beSAndroid Build Coastguard Worker struct xsk_ring_cons *comp_save;
59*053f45beSAndroid Build Coastguard Worker char *umem_area;
60*053f45beSAndroid Build Coastguard Worker struct xsk_umem_config config;
61*053f45beSAndroid Build Coastguard Worker int fd;
62*053f45beSAndroid Build Coastguard Worker int refcount;
63*053f45beSAndroid Build Coastguard Worker struct list_head ctx_list;
64*053f45beSAndroid Build Coastguard Worker bool rx_ring_setup_done;
65*053f45beSAndroid Build Coastguard Worker bool tx_ring_setup_done;
66*053f45beSAndroid Build Coastguard Worker };
67*053f45beSAndroid Build Coastguard Worker
68*053f45beSAndroid Build Coastguard Worker struct xsk_ctx {
69*053f45beSAndroid Build Coastguard Worker struct xsk_ring_prod *fill;
70*053f45beSAndroid Build Coastguard Worker struct xsk_ring_cons *comp;
71*053f45beSAndroid Build Coastguard Worker __u32 queue_id;
72*053f45beSAndroid Build Coastguard Worker struct xsk_umem *umem;
73*053f45beSAndroid Build Coastguard Worker int refcount;
74*053f45beSAndroid Build Coastguard Worker int ifindex;
75*053f45beSAndroid Build Coastguard Worker struct list_head list;
76*053f45beSAndroid Build Coastguard Worker int prog_fd;
77*053f45beSAndroid Build Coastguard Worker int link_fd;
78*053f45beSAndroid Build Coastguard Worker int xsks_map_fd;
79*053f45beSAndroid Build Coastguard Worker char ifname[IFNAMSIZ];
80*053f45beSAndroid Build Coastguard Worker bool has_bpf_link;
81*053f45beSAndroid Build Coastguard Worker };
82*053f45beSAndroid Build Coastguard Worker
83*053f45beSAndroid Build Coastguard Worker struct xsk_socket {
84*053f45beSAndroid Build Coastguard Worker struct xsk_ring_cons *rx;
85*053f45beSAndroid Build Coastguard Worker struct xsk_ring_prod *tx;
86*053f45beSAndroid Build Coastguard Worker __u64 outstanding_tx;
87*053f45beSAndroid Build Coastguard Worker struct xsk_ctx *ctx;
88*053f45beSAndroid Build Coastguard Worker struct xsk_socket_config config;
89*053f45beSAndroid Build Coastguard Worker int fd;
90*053f45beSAndroid Build Coastguard Worker };
91*053f45beSAndroid Build Coastguard Worker
92*053f45beSAndroid Build Coastguard Worker struct xsk_nl_info {
93*053f45beSAndroid Build Coastguard Worker bool xdp_prog_attached;
94*053f45beSAndroid Build Coastguard Worker int ifindex;
95*053f45beSAndroid Build Coastguard Worker int fd;
96*053f45beSAndroid Build Coastguard Worker };
97*053f45beSAndroid Build Coastguard Worker
98*053f45beSAndroid Build Coastguard Worker /* Up until and including Linux 5.3 */
99*053f45beSAndroid Build Coastguard Worker struct xdp_ring_offset_v1 {
100*053f45beSAndroid Build Coastguard Worker __u64 producer;
101*053f45beSAndroid Build Coastguard Worker __u64 consumer;
102*053f45beSAndroid Build Coastguard Worker __u64 desc;
103*053f45beSAndroid Build Coastguard Worker };
104*053f45beSAndroid Build Coastguard Worker
105*053f45beSAndroid Build Coastguard Worker /* Up until and including Linux 5.3 */
106*053f45beSAndroid Build Coastguard Worker struct xdp_mmap_offsets_v1 {
107*053f45beSAndroid Build Coastguard Worker struct xdp_ring_offset_v1 rx;
108*053f45beSAndroid Build Coastguard Worker struct xdp_ring_offset_v1 tx;
109*053f45beSAndroid Build Coastguard Worker struct xdp_ring_offset_v1 fr;
110*053f45beSAndroid Build Coastguard Worker struct xdp_ring_offset_v1 cr;
111*053f45beSAndroid Build Coastguard Worker };
112*053f45beSAndroid Build Coastguard Worker
xsk_umem__fd(const struct xsk_umem * umem)113*053f45beSAndroid Build Coastguard Worker int xsk_umem__fd(const struct xsk_umem *umem)
114*053f45beSAndroid Build Coastguard Worker {
115*053f45beSAndroid Build Coastguard Worker return umem ? umem->fd : -EINVAL;
116*053f45beSAndroid Build Coastguard Worker }
117*053f45beSAndroid Build Coastguard Worker
xsk_socket__fd(const struct xsk_socket * xsk)118*053f45beSAndroid Build Coastguard Worker int xsk_socket__fd(const struct xsk_socket *xsk)
119*053f45beSAndroid Build Coastguard Worker {
120*053f45beSAndroid Build Coastguard Worker return xsk ? xsk->fd : -EINVAL;
121*053f45beSAndroid Build Coastguard Worker }
122*053f45beSAndroid Build Coastguard Worker
xsk_page_aligned(void * buffer)123*053f45beSAndroid Build Coastguard Worker static bool xsk_page_aligned(void *buffer)
124*053f45beSAndroid Build Coastguard Worker {
125*053f45beSAndroid Build Coastguard Worker unsigned long addr = (unsigned long)buffer;
126*053f45beSAndroid Build Coastguard Worker
127*053f45beSAndroid Build Coastguard Worker return !(addr & (getpagesize() - 1));
128*053f45beSAndroid Build Coastguard Worker }
129*053f45beSAndroid Build Coastguard Worker
xsk_set_umem_config(struct xsk_umem_config * cfg,const struct xsk_umem_config * usr_cfg)130*053f45beSAndroid Build Coastguard Worker static void xsk_set_umem_config(struct xsk_umem_config *cfg,
131*053f45beSAndroid Build Coastguard Worker const struct xsk_umem_config *usr_cfg)
132*053f45beSAndroid Build Coastguard Worker {
133*053f45beSAndroid Build Coastguard Worker if (!usr_cfg) {
134*053f45beSAndroid Build Coastguard Worker cfg->fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
135*053f45beSAndroid Build Coastguard Worker cfg->comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS;
136*053f45beSAndroid Build Coastguard Worker cfg->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
137*053f45beSAndroid Build Coastguard Worker cfg->frame_headroom = XSK_UMEM__DEFAULT_FRAME_HEADROOM;
138*053f45beSAndroid Build Coastguard Worker cfg->flags = XSK_UMEM__DEFAULT_FLAGS;
139*053f45beSAndroid Build Coastguard Worker return;
140*053f45beSAndroid Build Coastguard Worker }
141*053f45beSAndroid Build Coastguard Worker
142*053f45beSAndroid Build Coastguard Worker cfg->fill_size = usr_cfg->fill_size;
143*053f45beSAndroid Build Coastguard Worker cfg->comp_size = usr_cfg->comp_size;
144*053f45beSAndroid Build Coastguard Worker cfg->frame_size = usr_cfg->frame_size;
145*053f45beSAndroid Build Coastguard Worker cfg->frame_headroom = usr_cfg->frame_headroom;
146*053f45beSAndroid Build Coastguard Worker cfg->flags = usr_cfg->flags;
147*053f45beSAndroid Build Coastguard Worker }
148*053f45beSAndroid Build Coastguard Worker
xsk_set_xdp_socket_config(struct xsk_socket_config * cfg,const struct xsk_socket_config * usr_cfg)149*053f45beSAndroid Build Coastguard Worker static int xsk_set_xdp_socket_config(struct xsk_socket_config *cfg,
150*053f45beSAndroid Build Coastguard Worker const struct xsk_socket_config *usr_cfg)
151*053f45beSAndroid Build Coastguard Worker {
152*053f45beSAndroid Build Coastguard Worker if (!usr_cfg) {
153*053f45beSAndroid Build Coastguard Worker cfg->rx_size = XSK_RING_CONS__DEFAULT_NUM_DESCS;
154*053f45beSAndroid Build Coastguard Worker cfg->tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
155*053f45beSAndroid Build Coastguard Worker cfg->libbpf_flags = 0;
156*053f45beSAndroid Build Coastguard Worker cfg->xdp_flags = 0;
157*053f45beSAndroid Build Coastguard Worker cfg->bind_flags = 0;
158*053f45beSAndroid Build Coastguard Worker return 0;
159*053f45beSAndroid Build Coastguard Worker }
160*053f45beSAndroid Build Coastguard Worker
161*053f45beSAndroid Build Coastguard Worker if (usr_cfg->libbpf_flags & ~XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD)
162*053f45beSAndroid Build Coastguard Worker return -EINVAL;
163*053f45beSAndroid Build Coastguard Worker
164*053f45beSAndroid Build Coastguard Worker cfg->rx_size = usr_cfg->rx_size;
165*053f45beSAndroid Build Coastguard Worker cfg->tx_size = usr_cfg->tx_size;
166*053f45beSAndroid Build Coastguard Worker cfg->libbpf_flags = usr_cfg->libbpf_flags;
167*053f45beSAndroid Build Coastguard Worker cfg->xdp_flags = usr_cfg->xdp_flags;
168*053f45beSAndroid Build Coastguard Worker cfg->bind_flags = usr_cfg->bind_flags;
169*053f45beSAndroid Build Coastguard Worker
170*053f45beSAndroid Build Coastguard Worker return 0;
171*053f45beSAndroid Build Coastguard Worker }
172*053f45beSAndroid Build Coastguard Worker
xsk_mmap_offsets_v1(struct xdp_mmap_offsets * off)173*053f45beSAndroid Build Coastguard Worker static void xsk_mmap_offsets_v1(struct xdp_mmap_offsets *off)
174*053f45beSAndroid Build Coastguard Worker {
175*053f45beSAndroid Build Coastguard Worker struct xdp_mmap_offsets_v1 off_v1;
176*053f45beSAndroid Build Coastguard Worker
177*053f45beSAndroid Build Coastguard Worker /* getsockopt on a kernel <= 5.3 has no flags fields.
178*053f45beSAndroid Build Coastguard Worker * Copy over the offsets to the correct places in the >=5.4 format
179*053f45beSAndroid Build Coastguard Worker * and put the flags where they would have been on that kernel.
180*053f45beSAndroid Build Coastguard Worker */
181*053f45beSAndroid Build Coastguard Worker memcpy(&off_v1, off, sizeof(off_v1));
182*053f45beSAndroid Build Coastguard Worker
183*053f45beSAndroid Build Coastguard Worker off->rx.producer = off_v1.rx.producer;
184*053f45beSAndroid Build Coastguard Worker off->rx.consumer = off_v1.rx.consumer;
185*053f45beSAndroid Build Coastguard Worker off->rx.desc = off_v1.rx.desc;
186*053f45beSAndroid Build Coastguard Worker off->rx.flags = off_v1.rx.consumer + sizeof(__u32);
187*053f45beSAndroid Build Coastguard Worker
188*053f45beSAndroid Build Coastguard Worker off->tx.producer = off_v1.tx.producer;
189*053f45beSAndroid Build Coastguard Worker off->tx.consumer = off_v1.tx.consumer;
190*053f45beSAndroid Build Coastguard Worker off->tx.desc = off_v1.tx.desc;
191*053f45beSAndroid Build Coastguard Worker off->tx.flags = off_v1.tx.consumer + sizeof(__u32);
192*053f45beSAndroid Build Coastguard Worker
193*053f45beSAndroid Build Coastguard Worker off->fr.producer = off_v1.fr.producer;
194*053f45beSAndroid Build Coastguard Worker off->fr.consumer = off_v1.fr.consumer;
195*053f45beSAndroid Build Coastguard Worker off->fr.desc = off_v1.fr.desc;
196*053f45beSAndroid Build Coastguard Worker off->fr.flags = off_v1.fr.consumer + sizeof(__u32);
197*053f45beSAndroid Build Coastguard Worker
198*053f45beSAndroid Build Coastguard Worker off->cr.producer = off_v1.cr.producer;
199*053f45beSAndroid Build Coastguard Worker off->cr.consumer = off_v1.cr.consumer;
200*053f45beSAndroid Build Coastguard Worker off->cr.desc = off_v1.cr.desc;
201*053f45beSAndroid Build Coastguard Worker off->cr.flags = off_v1.cr.consumer + sizeof(__u32);
202*053f45beSAndroid Build Coastguard Worker }
203*053f45beSAndroid Build Coastguard Worker
xsk_get_mmap_offsets(int fd,struct xdp_mmap_offsets * off)204*053f45beSAndroid Build Coastguard Worker static int xsk_get_mmap_offsets(int fd, struct xdp_mmap_offsets *off)
205*053f45beSAndroid Build Coastguard Worker {
206*053f45beSAndroid Build Coastguard Worker socklen_t optlen;
207*053f45beSAndroid Build Coastguard Worker int err;
208*053f45beSAndroid Build Coastguard Worker
209*053f45beSAndroid Build Coastguard Worker optlen = sizeof(*off);
210*053f45beSAndroid Build Coastguard Worker err = getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, off, &optlen);
211*053f45beSAndroid Build Coastguard Worker if (err)
212*053f45beSAndroid Build Coastguard Worker return err;
213*053f45beSAndroid Build Coastguard Worker
214*053f45beSAndroid Build Coastguard Worker if (optlen == sizeof(*off))
215*053f45beSAndroid Build Coastguard Worker return 0;
216*053f45beSAndroid Build Coastguard Worker
217*053f45beSAndroid Build Coastguard Worker if (optlen == sizeof(struct xdp_mmap_offsets_v1)) {
218*053f45beSAndroid Build Coastguard Worker xsk_mmap_offsets_v1(off);
219*053f45beSAndroid Build Coastguard Worker return 0;
220*053f45beSAndroid Build Coastguard Worker }
221*053f45beSAndroid Build Coastguard Worker
222*053f45beSAndroid Build Coastguard Worker return -EINVAL;
223*053f45beSAndroid Build Coastguard Worker }
224*053f45beSAndroid Build Coastguard Worker
xsk_create_umem_rings(struct xsk_umem * umem,int fd,struct xsk_ring_prod * fill,struct xsk_ring_cons * comp)225*053f45beSAndroid Build Coastguard Worker static int xsk_create_umem_rings(struct xsk_umem *umem, int fd,
226*053f45beSAndroid Build Coastguard Worker struct xsk_ring_prod *fill,
227*053f45beSAndroid Build Coastguard Worker struct xsk_ring_cons *comp)
228*053f45beSAndroid Build Coastguard Worker {
229*053f45beSAndroid Build Coastguard Worker struct xdp_mmap_offsets off;
230*053f45beSAndroid Build Coastguard Worker void *map;
231*053f45beSAndroid Build Coastguard Worker int err;
232*053f45beSAndroid Build Coastguard Worker
233*053f45beSAndroid Build Coastguard Worker err = setsockopt(fd, SOL_XDP, XDP_UMEM_FILL_RING,
234*053f45beSAndroid Build Coastguard Worker &umem->config.fill_size,
235*053f45beSAndroid Build Coastguard Worker sizeof(umem->config.fill_size));
236*053f45beSAndroid Build Coastguard Worker if (err)
237*053f45beSAndroid Build Coastguard Worker return -errno;
238*053f45beSAndroid Build Coastguard Worker
239*053f45beSAndroid Build Coastguard Worker err = setsockopt(fd, SOL_XDP, XDP_UMEM_COMPLETION_RING,
240*053f45beSAndroid Build Coastguard Worker &umem->config.comp_size,
241*053f45beSAndroid Build Coastguard Worker sizeof(umem->config.comp_size));
242*053f45beSAndroid Build Coastguard Worker if (err)
243*053f45beSAndroid Build Coastguard Worker return -errno;
244*053f45beSAndroid Build Coastguard Worker
245*053f45beSAndroid Build Coastguard Worker err = xsk_get_mmap_offsets(fd, &off);
246*053f45beSAndroid Build Coastguard Worker if (err)
247*053f45beSAndroid Build Coastguard Worker return -errno;
248*053f45beSAndroid Build Coastguard Worker
249*053f45beSAndroid Build Coastguard Worker map = mmap(NULL, off.fr.desc + umem->config.fill_size * sizeof(__u64),
250*053f45beSAndroid Build Coastguard Worker PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
251*053f45beSAndroid Build Coastguard Worker XDP_UMEM_PGOFF_FILL_RING);
252*053f45beSAndroid Build Coastguard Worker if (map == MAP_FAILED)
253*053f45beSAndroid Build Coastguard Worker return -errno;
254*053f45beSAndroid Build Coastguard Worker
255*053f45beSAndroid Build Coastguard Worker fill->mask = umem->config.fill_size - 1;
256*053f45beSAndroid Build Coastguard Worker fill->size = umem->config.fill_size;
257*053f45beSAndroid Build Coastguard Worker fill->producer = map + off.fr.producer;
258*053f45beSAndroid Build Coastguard Worker fill->consumer = map + off.fr.consumer;
259*053f45beSAndroid Build Coastguard Worker fill->flags = map + off.fr.flags;
260*053f45beSAndroid Build Coastguard Worker fill->ring = map + off.fr.desc;
261*053f45beSAndroid Build Coastguard Worker fill->cached_cons = umem->config.fill_size;
262*053f45beSAndroid Build Coastguard Worker
263*053f45beSAndroid Build Coastguard Worker map = mmap(NULL, off.cr.desc + umem->config.comp_size * sizeof(__u64),
264*053f45beSAndroid Build Coastguard Worker PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
265*053f45beSAndroid Build Coastguard Worker XDP_UMEM_PGOFF_COMPLETION_RING);
266*053f45beSAndroid Build Coastguard Worker if (map == MAP_FAILED) {
267*053f45beSAndroid Build Coastguard Worker err = -errno;
268*053f45beSAndroid Build Coastguard Worker goto out_mmap;
269*053f45beSAndroid Build Coastguard Worker }
270*053f45beSAndroid Build Coastguard Worker
271*053f45beSAndroid Build Coastguard Worker comp->mask = umem->config.comp_size - 1;
272*053f45beSAndroid Build Coastguard Worker comp->size = umem->config.comp_size;
273*053f45beSAndroid Build Coastguard Worker comp->producer = map + off.cr.producer;
274*053f45beSAndroid Build Coastguard Worker comp->consumer = map + off.cr.consumer;
275*053f45beSAndroid Build Coastguard Worker comp->flags = map + off.cr.flags;
276*053f45beSAndroid Build Coastguard Worker comp->ring = map + off.cr.desc;
277*053f45beSAndroid Build Coastguard Worker
278*053f45beSAndroid Build Coastguard Worker return 0;
279*053f45beSAndroid Build Coastguard Worker
280*053f45beSAndroid Build Coastguard Worker out_mmap:
281*053f45beSAndroid Build Coastguard Worker munmap(map, off.fr.desc + umem->config.fill_size * sizeof(__u64));
282*053f45beSAndroid Build Coastguard Worker return err;
283*053f45beSAndroid Build Coastguard Worker }
284*053f45beSAndroid Build Coastguard Worker
xsk_umem__create(struct xsk_umem ** umem_ptr,void * umem_area,__u64 size,struct xsk_ring_prod * fill,struct xsk_ring_cons * comp,const struct xsk_umem_config * usr_config)285*053f45beSAndroid Build Coastguard Worker int xsk_umem__create(struct xsk_umem **umem_ptr, void *umem_area,
286*053f45beSAndroid Build Coastguard Worker __u64 size, struct xsk_ring_prod *fill,
287*053f45beSAndroid Build Coastguard Worker struct xsk_ring_cons *comp,
288*053f45beSAndroid Build Coastguard Worker const struct xsk_umem_config *usr_config)
289*053f45beSAndroid Build Coastguard Worker {
290*053f45beSAndroid Build Coastguard Worker struct xdp_umem_reg mr;
291*053f45beSAndroid Build Coastguard Worker struct xsk_umem *umem;
292*053f45beSAndroid Build Coastguard Worker int err;
293*053f45beSAndroid Build Coastguard Worker
294*053f45beSAndroid Build Coastguard Worker if (!umem_area || !umem_ptr || !fill || !comp)
295*053f45beSAndroid Build Coastguard Worker return -EFAULT;
296*053f45beSAndroid Build Coastguard Worker if (!size && !xsk_page_aligned(umem_area))
297*053f45beSAndroid Build Coastguard Worker return -EINVAL;
298*053f45beSAndroid Build Coastguard Worker
299*053f45beSAndroid Build Coastguard Worker umem = calloc(1, sizeof(*umem));
300*053f45beSAndroid Build Coastguard Worker if (!umem)
301*053f45beSAndroid Build Coastguard Worker return -ENOMEM;
302*053f45beSAndroid Build Coastguard Worker
303*053f45beSAndroid Build Coastguard Worker umem->fd = socket(AF_XDP, SOCK_RAW | SOCK_CLOEXEC, 0);
304*053f45beSAndroid Build Coastguard Worker if (umem->fd < 0) {
305*053f45beSAndroid Build Coastguard Worker err = -errno;
306*053f45beSAndroid Build Coastguard Worker goto out_umem_alloc;
307*053f45beSAndroid Build Coastguard Worker }
308*053f45beSAndroid Build Coastguard Worker
309*053f45beSAndroid Build Coastguard Worker umem->umem_area = umem_area;
310*053f45beSAndroid Build Coastguard Worker INIT_LIST_HEAD(&umem->ctx_list);
311*053f45beSAndroid Build Coastguard Worker xsk_set_umem_config(&umem->config, usr_config);
312*053f45beSAndroid Build Coastguard Worker
313*053f45beSAndroid Build Coastguard Worker memset(&mr, 0, sizeof(mr));
314*053f45beSAndroid Build Coastguard Worker mr.addr = (uintptr_t)umem_area;
315*053f45beSAndroid Build Coastguard Worker mr.len = size;
316*053f45beSAndroid Build Coastguard Worker mr.chunk_size = umem->config.frame_size;
317*053f45beSAndroid Build Coastguard Worker mr.headroom = umem->config.frame_headroom;
318*053f45beSAndroid Build Coastguard Worker mr.flags = umem->config.flags;
319*053f45beSAndroid Build Coastguard Worker
320*053f45beSAndroid Build Coastguard Worker err = setsockopt(umem->fd, SOL_XDP, XDP_UMEM_REG, &mr, sizeof(mr));
321*053f45beSAndroid Build Coastguard Worker if (err) {
322*053f45beSAndroid Build Coastguard Worker err = -errno;
323*053f45beSAndroid Build Coastguard Worker goto out_socket;
324*053f45beSAndroid Build Coastguard Worker }
325*053f45beSAndroid Build Coastguard Worker
326*053f45beSAndroid Build Coastguard Worker err = xsk_create_umem_rings(umem, umem->fd, fill, comp);
327*053f45beSAndroid Build Coastguard Worker if (err)
328*053f45beSAndroid Build Coastguard Worker goto out_socket;
329*053f45beSAndroid Build Coastguard Worker
330*053f45beSAndroid Build Coastguard Worker umem->fill_save = fill;
331*053f45beSAndroid Build Coastguard Worker umem->comp_save = comp;
332*053f45beSAndroid Build Coastguard Worker *umem_ptr = umem;
333*053f45beSAndroid Build Coastguard Worker return 0;
334*053f45beSAndroid Build Coastguard Worker
335*053f45beSAndroid Build Coastguard Worker out_socket:
336*053f45beSAndroid Build Coastguard Worker close(umem->fd);
337*053f45beSAndroid Build Coastguard Worker out_umem_alloc:
338*053f45beSAndroid Build Coastguard Worker free(umem);
339*053f45beSAndroid Build Coastguard Worker return err;
340*053f45beSAndroid Build Coastguard Worker }
341*053f45beSAndroid Build Coastguard Worker
342*053f45beSAndroid Build Coastguard Worker struct xsk_umem_config_v1 {
343*053f45beSAndroid Build Coastguard Worker __u32 fill_size;
344*053f45beSAndroid Build Coastguard Worker __u32 comp_size;
345*053f45beSAndroid Build Coastguard Worker __u32 frame_size;
346*053f45beSAndroid Build Coastguard Worker __u32 frame_headroom;
347*053f45beSAndroid Build Coastguard Worker };
348*053f45beSAndroid Build Coastguard Worker
get_xsk_prog(void)349*053f45beSAndroid Build Coastguard Worker static enum xsk_prog get_xsk_prog(void)
350*053f45beSAndroid Build Coastguard Worker {
351*053f45beSAndroid Build Coastguard Worker enum xsk_prog detected = XSK_PROG_FALLBACK;
352*053f45beSAndroid Build Coastguard Worker char data_in = 0, data_out;
353*053f45beSAndroid Build Coastguard Worker struct bpf_insn insns[] = {
354*053f45beSAndroid Build Coastguard Worker BPF_LD_MAP_FD(BPF_REG_1, 0),
355*053f45beSAndroid Build Coastguard Worker BPF_MOV64_IMM(BPF_REG_2, 0),
356*053f45beSAndroid Build Coastguard Worker BPF_MOV64_IMM(BPF_REG_3, XDP_PASS),
357*053f45beSAndroid Build Coastguard Worker BPF_EMIT_CALL(BPF_FUNC_redirect_map),
358*053f45beSAndroid Build Coastguard Worker BPF_EXIT_INSN(),
359*053f45beSAndroid Build Coastguard Worker };
360*053f45beSAndroid Build Coastguard Worker LIBBPF_OPTS(bpf_test_run_opts, opts,
361*053f45beSAndroid Build Coastguard Worker .data_in = &data_in,
362*053f45beSAndroid Build Coastguard Worker .data_size_in = 1,
363*053f45beSAndroid Build Coastguard Worker .data_out = &data_out,
364*053f45beSAndroid Build Coastguard Worker );
365*053f45beSAndroid Build Coastguard Worker
366*053f45beSAndroid Build Coastguard Worker int prog_fd, map_fd, ret, insn_cnt = ARRAY_SIZE(insns);
367*053f45beSAndroid Build Coastguard Worker
368*053f45beSAndroid Build Coastguard Worker map_fd = bpf_map_create(BPF_MAP_TYPE_XSKMAP, NULL, sizeof(int), sizeof(int), 1, NULL);
369*053f45beSAndroid Build Coastguard Worker if (map_fd < 0)
370*053f45beSAndroid Build Coastguard Worker return detected;
371*053f45beSAndroid Build Coastguard Worker
372*053f45beSAndroid Build Coastguard Worker insns[0].imm = map_fd;
373*053f45beSAndroid Build Coastguard Worker
374*053f45beSAndroid Build Coastguard Worker prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL);
375*053f45beSAndroid Build Coastguard Worker if (prog_fd < 0) {
376*053f45beSAndroid Build Coastguard Worker close(map_fd);
377*053f45beSAndroid Build Coastguard Worker return detected;
378*053f45beSAndroid Build Coastguard Worker }
379*053f45beSAndroid Build Coastguard Worker
380*053f45beSAndroid Build Coastguard Worker ret = bpf_prog_test_run_opts(prog_fd, &opts);
381*053f45beSAndroid Build Coastguard Worker if (!ret && opts.retval == XDP_PASS)
382*053f45beSAndroid Build Coastguard Worker detected = XSK_PROG_REDIRECT_FLAGS;
383*053f45beSAndroid Build Coastguard Worker close(prog_fd);
384*053f45beSAndroid Build Coastguard Worker close(map_fd);
385*053f45beSAndroid Build Coastguard Worker return detected;
386*053f45beSAndroid Build Coastguard Worker }
387*053f45beSAndroid Build Coastguard Worker
xsk_load_xdp_prog(struct xsk_socket * xsk)388*053f45beSAndroid Build Coastguard Worker static int xsk_load_xdp_prog(struct xsk_socket *xsk)
389*053f45beSAndroid Build Coastguard Worker {
390*053f45beSAndroid Build Coastguard Worker static const int log_buf_size = 16 * 1024;
391*053f45beSAndroid Build Coastguard Worker struct xsk_ctx *ctx = xsk->ctx;
392*053f45beSAndroid Build Coastguard Worker char log_buf[log_buf_size];
393*053f45beSAndroid Build Coastguard Worker int prog_fd;
394*053f45beSAndroid Build Coastguard Worker
395*053f45beSAndroid Build Coastguard Worker /* This is the fallback C-program:
396*053f45beSAndroid Build Coastguard Worker * SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)
397*053f45beSAndroid Build Coastguard Worker * {
398*053f45beSAndroid Build Coastguard Worker * int ret, index = ctx->rx_queue_index;
399*053f45beSAndroid Build Coastguard Worker *
400*053f45beSAndroid Build Coastguard Worker * // A set entry here means that the correspnding queue_id
401*053f45beSAndroid Build Coastguard Worker * // has an active AF_XDP socket bound to it.
402*053f45beSAndroid Build Coastguard Worker * ret = bpf_redirect_map(&xsks_map, index, XDP_PASS);
403*053f45beSAndroid Build Coastguard Worker * if (ret > 0)
404*053f45beSAndroid Build Coastguard Worker * return ret;
405*053f45beSAndroid Build Coastguard Worker *
406*053f45beSAndroid Build Coastguard Worker * // Fallback for pre-5.3 kernels, not supporting default
407*053f45beSAndroid Build Coastguard Worker * // action in the flags parameter.
408*053f45beSAndroid Build Coastguard Worker * if (bpf_map_lookup_elem(&xsks_map, &index))
409*053f45beSAndroid Build Coastguard Worker * return bpf_redirect_map(&xsks_map, index, 0);
410*053f45beSAndroid Build Coastguard Worker * return XDP_PASS;
411*053f45beSAndroid Build Coastguard Worker * }
412*053f45beSAndroid Build Coastguard Worker */
413*053f45beSAndroid Build Coastguard Worker struct bpf_insn prog[] = {
414*053f45beSAndroid Build Coastguard Worker /* r2 = *(u32 *)(r1 + 16) */
415*053f45beSAndroid Build Coastguard Worker BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 16),
416*053f45beSAndroid Build Coastguard Worker /* *(u32 *)(r10 - 4) = r2 */
417*053f45beSAndroid Build Coastguard Worker BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_2, -4),
418*053f45beSAndroid Build Coastguard Worker /* r1 = xskmap[] */
419*053f45beSAndroid Build Coastguard Worker BPF_LD_MAP_FD(BPF_REG_1, ctx->xsks_map_fd),
420*053f45beSAndroid Build Coastguard Worker /* r3 = XDP_PASS */
421*053f45beSAndroid Build Coastguard Worker BPF_MOV64_IMM(BPF_REG_3, 2),
422*053f45beSAndroid Build Coastguard Worker /* call bpf_redirect_map */
423*053f45beSAndroid Build Coastguard Worker BPF_EMIT_CALL(BPF_FUNC_redirect_map),
424*053f45beSAndroid Build Coastguard Worker /* if w0 != 0 goto pc+13 */
425*053f45beSAndroid Build Coastguard Worker BPF_JMP32_IMM(BPF_JSGT, BPF_REG_0, 0, 13),
426*053f45beSAndroid Build Coastguard Worker /* r2 = r10 */
427*053f45beSAndroid Build Coastguard Worker BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
428*053f45beSAndroid Build Coastguard Worker /* r2 += -4 */
429*053f45beSAndroid Build Coastguard Worker BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4),
430*053f45beSAndroid Build Coastguard Worker /* r1 = xskmap[] */
431*053f45beSAndroid Build Coastguard Worker BPF_LD_MAP_FD(BPF_REG_1, ctx->xsks_map_fd),
432*053f45beSAndroid Build Coastguard Worker /* call bpf_map_lookup_elem */
433*053f45beSAndroid Build Coastguard Worker BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
434*053f45beSAndroid Build Coastguard Worker /* r1 = r0 */
435*053f45beSAndroid Build Coastguard Worker BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
436*053f45beSAndroid Build Coastguard Worker /* r0 = XDP_PASS */
437*053f45beSAndroid Build Coastguard Worker BPF_MOV64_IMM(BPF_REG_0, 2),
438*053f45beSAndroid Build Coastguard Worker /* if r1 == 0 goto pc+5 */
439*053f45beSAndroid Build Coastguard Worker BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 5),
440*053f45beSAndroid Build Coastguard Worker /* r2 = *(u32 *)(r10 - 4) */
441*053f45beSAndroid Build Coastguard Worker BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_10, -4),
442*053f45beSAndroid Build Coastguard Worker /* r1 = xskmap[] */
443*053f45beSAndroid Build Coastguard Worker BPF_LD_MAP_FD(BPF_REG_1, ctx->xsks_map_fd),
444*053f45beSAndroid Build Coastguard Worker /* r3 = 0 */
445*053f45beSAndroid Build Coastguard Worker BPF_MOV64_IMM(BPF_REG_3, 0),
446*053f45beSAndroid Build Coastguard Worker /* call bpf_redirect_map */
447*053f45beSAndroid Build Coastguard Worker BPF_EMIT_CALL(BPF_FUNC_redirect_map),
448*053f45beSAndroid Build Coastguard Worker /* The jumps are to this instruction */
449*053f45beSAndroid Build Coastguard Worker BPF_EXIT_INSN(),
450*053f45beSAndroid Build Coastguard Worker };
451*053f45beSAndroid Build Coastguard Worker
452*053f45beSAndroid Build Coastguard Worker /* This is the post-5.3 kernel C-program:
453*053f45beSAndroid Build Coastguard Worker * SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)
454*053f45beSAndroid Build Coastguard Worker * {
455*053f45beSAndroid Build Coastguard Worker * return bpf_redirect_map(&xsks_map, ctx->rx_queue_index, XDP_PASS);
456*053f45beSAndroid Build Coastguard Worker * }
457*053f45beSAndroid Build Coastguard Worker */
458*053f45beSAndroid Build Coastguard Worker struct bpf_insn prog_redirect_flags[] = {
459*053f45beSAndroid Build Coastguard Worker /* r2 = *(u32 *)(r1 + 16) */
460*053f45beSAndroid Build Coastguard Worker BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 16),
461*053f45beSAndroid Build Coastguard Worker /* r1 = xskmap[] */
462*053f45beSAndroid Build Coastguard Worker BPF_LD_MAP_FD(BPF_REG_1, ctx->xsks_map_fd),
463*053f45beSAndroid Build Coastguard Worker /* r3 = XDP_PASS */
464*053f45beSAndroid Build Coastguard Worker BPF_MOV64_IMM(BPF_REG_3, 2),
465*053f45beSAndroid Build Coastguard Worker /* call bpf_redirect_map */
466*053f45beSAndroid Build Coastguard Worker BPF_EMIT_CALL(BPF_FUNC_redirect_map),
467*053f45beSAndroid Build Coastguard Worker BPF_EXIT_INSN(),
468*053f45beSAndroid Build Coastguard Worker };
469*053f45beSAndroid Build Coastguard Worker size_t insns_cnt[] = {ARRAY_SIZE(prog),
470*053f45beSAndroid Build Coastguard Worker ARRAY_SIZE(prog_redirect_flags),
471*053f45beSAndroid Build Coastguard Worker };
472*053f45beSAndroid Build Coastguard Worker struct bpf_insn *progs[] = {prog, prog_redirect_flags};
473*053f45beSAndroid Build Coastguard Worker enum xsk_prog option = get_xsk_prog();
474*053f45beSAndroid Build Coastguard Worker LIBBPF_OPTS(bpf_prog_load_opts, opts,
475*053f45beSAndroid Build Coastguard Worker .log_buf = log_buf,
476*053f45beSAndroid Build Coastguard Worker .log_size = log_buf_size,
477*053f45beSAndroid Build Coastguard Worker );
478*053f45beSAndroid Build Coastguard Worker
479*053f45beSAndroid Build Coastguard Worker prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "LGPL-2.1 or BSD-2-Clause",
480*053f45beSAndroid Build Coastguard Worker progs[option], insns_cnt[option], &opts);
481*053f45beSAndroid Build Coastguard Worker if (prog_fd < 0) {
482*053f45beSAndroid Build Coastguard Worker pr_warn("BPF log buffer:\n%s", log_buf);
483*053f45beSAndroid Build Coastguard Worker return prog_fd;
484*053f45beSAndroid Build Coastguard Worker }
485*053f45beSAndroid Build Coastguard Worker
486*053f45beSAndroid Build Coastguard Worker ctx->prog_fd = prog_fd;
487*053f45beSAndroid Build Coastguard Worker return 0;
488*053f45beSAndroid Build Coastguard Worker }
489*053f45beSAndroid Build Coastguard Worker
xsk_create_bpf_link(struct xsk_socket * xsk)490*053f45beSAndroid Build Coastguard Worker static int xsk_create_bpf_link(struct xsk_socket *xsk)
491*053f45beSAndroid Build Coastguard Worker {
492*053f45beSAndroid Build Coastguard Worker DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts);
493*053f45beSAndroid Build Coastguard Worker struct xsk_ctx *ctx = xsk->ctx;
494*053f45beSAndroid Build Coastguard Worker __u32 prog_id = 0;
495*053f45beSAndroid Build Coastguard Worker int link_fd;
496*053f45beSAndroid Build Coastguard Worker int err;
497*053f45beSAndroid Build Coastguard Worker
498*053f45beSAndroid Build Coastguard Worker err = bpf_xdp_query_id(ctx->ifindex, xsk->config.xdp_flags, &prog_id);
499*053f45beSAndroid Build Coastguard Worker if (err) {
500*053f45beSAndroid Build Coastguard Worker pr_warn("getting XDP prog id failed\n");
501*053f45beSAndroid Build Coastguard Worker return err;
502*053f45beSAndroid Build Coastguard Worker }
503*053f45beSAndroid Build Coastguard Worker
504*053f45beSAndroid Build Coastguard Worker /* if there's a netlink-based XDP prog loaded on interface, bail out
505*053f45beSAndroid Build Coastguard Worker * and ask user to do the removal by himself
506*053f45beSAndroid Build Coastguard Worker */
507*053f45beSAndroid Build Coastguard Worker if (prog_id) {
508*053f45beSAndroid Build Coastguard Worker pr_warn("Netlink-based XDP prog detected, please unload it in order to launch AF_XDP prog\n");
509*053f45beSAndroid Build Coastguard Worker return -EINVAL;
510*053f45beSAndroid Build Coastguard Worker }
511*053f45beSAndroid Build Coastguard Worker
512*053f45beSAndroid Build Coastguard Worker opts.flags = xsk->config.xdp_flags & ~(XDP_FLAGS_UPDATE_IF_NOEXIST | XDP_FLAGS_REPLACE);
513*053f45beSAndroid Build Coastguard Worker
514*053f45beSAndroid Build Coastguard Worker link_fd = bpf_link_create(ctx->prog_fd, ctx->ifindex, BPF_XDP, &opts);
515*053f45beSAndroid Build Coastguard Worker if (link_fd < 0) {
516*053f45beSAndroid Build Coastguard Worker pr_warn("bpf_link_create failed: %s\n", strerror(errno));
517*053f45beSAndroid Build Coastguard Worker return link_fd;
518*053f45beSAndroid Build Coastguard Worker }
519*053f45beSAndroid Build Coastguard Worker
520*053f45beSAndroid Build Coastguard Worker ctx->link_fd = link_fd;
521*053f45beSAndroid Build Coastguard Worker return 0;
522*053f45beSAndroid Build Coastguard Worker }
523*053f45beSAndroid Build Coastguard Worker
524*053f45beSAndroid Build Coastguard Worker /* Copy up to sz - 1 bytes from zero-terminated src string and ensure that dst
525*053f45beSAndroid Build Coastguard Worker * is zero-terminated string no matter what (unless sz == 0, in which case
526*053f45beSAndroid Build Coastguard Worker * it's a no-op). It's conceptually close to FreeBSD's strlcpy(), but differs
527*053f45beSAndroid Build Coastguard Worker * in what is returned. Given this is internal helper, it's trivial to extend
528*053f45beSAndroid Build Coastguard Worker * this, when necessary. Use this instead of strncpy inside libbpf source code.
529*053f45beSAndroid Build Coastguard Worker */
libbpf_strlcpy(char * dst,const char * src,size_t sz)530*053f45beSAndroid Build Coastguard Worker static inline void libbpf_strlcpy(char *dst, const char *src, size_t sz)
531*053f45beSAndroid Build Coastguard Worker {
532*053f45beSAndroid Build Coastguard Worker size_t i;
533*053f45beSAndroid Build Coastguard Worker
534*053f45beSAndroid Build Coastguard Worker if (sz == 0)
535*053f45beSAndroid Build Coastguard Worker return;
536*053f45beSAndroid Build Coastguard Worker
537*053f45beSAndroid Build Coastguard Worker sz--;
538*053f45beSAndroid Build Coastguard Worker for (i = 0; i < sz && src[i]; i++)
539*053f45beSAndroid Build Coastguard Worker dst[i] = src[i];
540*053f45beSAndroid Build Coastguard Worker dst[i] = '\0';
541*053f45beSAndroid Build Coastguard Worker }
542*053f45beSAndroid Build Coastguard Worker
xsk_get_max_queues(struct xsk_socket * xsk)543*053f45beSAndroid Build Coastguard Worker static int xsk_get_max_queues(struct xsk_socket *xsk)
544*053f45beSAndroid Build Coastguard Worker {
545*053f45beSAndroid Build Coastguard Worker struct ethtool_channels channels = { .cmd = ETHTOOL_GCHANNELS };
546*053f45beSAndroid Build Coastguard Worker struct xsk_ctx *ctx = xsk->ctx;
547*053f45beSAndroid Build Coastguard Worker struct ifreq ifr = {};
548*053f45beSAndroid Build Coastguard Worker int fd, err, ret;
549*053f45beSAndroid Build Coastguard Worker
550*053f45beSAndroid Build Coastguard Worker fd = socket(AF_LOCAL, SOCK_DGRAM | SOCK_CLOEXEC, 0);
551*053f45beSAndroid Build Coastguard Worker if (fd < 0)
552*053f45beSAndroid Build Coastguard Worker return -errno;
553*053f45beSAndroid Build Coastguard Worker
554*053f45beSAndroid Build Coastguard Worker ifr.ifr_data = (void *)&channels;
555*053f45beSAndroid Build Coastguard Worker libbpf_strlcpy(ifr.ifr_name, ctx->ifname, IFNAMSIZ);
556*053f45beSAndroid Build Coastguard Worker err = ioctl(fd, SIOCETHTOOL, &ifr);
557*053f45beSAndroid Build Coastguard Worker if (err && errno != EOPNOTSUPP) {
558*053f45beSAndroid Build Coastguard Worker ret = -errno;
559*053f45beSAndroid Build Coastguard Worker goto out;
560*053f45beSAndroid Build Coastguard Worker }
561*053f45beSAndroid Build Coastguard Worker
562*053f45beSAndroid Build Coastguard Worker if (err) {
563*053f45beSAndroid Build Coastguard Worker /* If the device says it has no channels, then all traffic
564*053f45beSAndroid Build Coastguard Worker * is sent to a single stream, so max queues = 1.
565*053f45beSAndroid Build Coastguard Worker */
566*053f45beSAndroid Build Coastguard Worker ret = 1;
567*053f45beSAndroid Build Coastguard Worker } else {
568*053f45beSAndroid Build Coastguard Worker /* Take the max of rx, tx, combined. Drivers return
569*053f45beSAndroid Build Coastguard Worker * the number of channels in different ways.
570*053f45beSAndroid Build Coastguard Worker */
571*053f45beSAndroid Build Coastguard Worker ret = max(channels.max_rx, channels.max_tx);
572*053f45beSAndroid Build Coastguard Worker ret = max(ret, (int)channels.max_combined);
573*053f45beSAndroid Build Coastguard Worker }
574*053f45beSAndroid Build Coastguard Worker
575*053f45beSAndroid Build Coastguard Worker out:
576*053f45beSAndroid Build Coastguard Worker close(fd);
577*053f45beSAndroid Build Coastguard Worker return ret;
578*053f45beSAndroid Build Coastguard Worker }
579*053f45beSAndroid Build Coastguard Worker
xsk_create_bpf_maps(struct xsk_socket * xsk)580*053f45beSAndroid Build Coastguard Worker static int xsk_create_bpf_maps(struct xsk_socket *xsk)
581*053f45beSAndroid Build Coastguard Worker {
582*053f45beSAndroid Build Coastguard Worker struct xsk_ctx *ctx = xsk->ctx;
583*053f45beSAndroid Build Coastguard Worker int max_queues;
584*053f45beSAndroid Build Coastguard Worker int fd;
585*053f45beSAndroid Build Coastguard Worker
586*053f45beSAndroid Build Coastguard Worker max_queues = xsk_get_max_queues(xsk);
587*053f45beSAndroid Build Coastguard Worker if (max_queues < 0)
588*053f45beSAndroid Build Coastguard Worker return max_queues;
589*053f45beSAndroid Build Coastguard Worker
590*053f45beSAndroid Build Coastguard Worker fd = bpf_map_create(BPF_MAP_TYPE_XSKMAP, "xsks_map",
591*053f45beSAndroid Build Coastguard Worker sizeof(int), sizeof(int), max_queues, NULL);
592*053f45beSAndroid Build Coastguard Worker if (fd < 0)
593*053f45beSAndroid Build Coastguard Worker return fd;
594*053f45beSAndroid Build Coastguard Worker
595*053f45beSAndroid Build Coastguard Worker ctx->xsks_map_fd = fd;
596*053f45beSAndroid Build Coastguard Worker
597*053f45beSAndroid Build Coastguard Worker return 0;
598*053f45beSAndroid Build Coastguard Worker }
599*053f45beSAndroid Build Coastguard Worker
xsk_delete_bpf_maps(struct xsk_socket * xsk)600*053f45beSAndroid Build Coastguard Worker static void xsk_delete_bpf_maps(struct xsk_socket *xsk)
601*053f45beSAndroid Build Coastguard Worker {
602*053f45beSAndroid Build Coastguard Worker struct xsk_ctx *ctx = xsk->ctx;
603*053f45beSAndroid Build Coastguard Worker
604*053f45beSAndroid Build Coastguard Worker bpf_map_delete_elem(ctx->xsks_map_fd, &ctx->queue_id);
605*053f45beSAndroid Build Coastguard Worker close(ctx->xsks_map_fd);
606*053f45beSAndroid Build Coastguard Worker }
607*053f45beSAndroid Build Coastguard Worker
xsk_lookup_bpf_maps(struct xsk_socket * xsk)608*053f45beSAndroid Build Coastguard Worker static int xsk_lookup_bpf_maps(struct xsk_socket *xsk)
609*053f45beSAndroid Build Coastguard Worker {
610*053f45beSAndroid Build Coastguard Worker __u32 i, *map_ids, num_maps, prog_len = sizeof(struct bpf_prog_info);
611*053f45beSAndroid Build Coastguard Worker __u32 map_len = sizeof(struct bpf_map_info);
612*053f45beSAndroid Build Coastguard Worker struct bpf_prog_info prog_info = {};
613*053f45beSAndroid Build Coastguard Worker struct xsk_ctx *ctx = xsk->ctx;
614*053f45beSAndroid Build Coastguard Worker struct bpf_map_info map_info;
615*053f45beSAndroid Build Coastguard Worker int fd, err;
616*053f45beSAndroid Build Coastguard Worker
617*053f45beSAndroid Build Coastguard Worker err = bpf_obj_get_info_by_fd(ctx->prog_fd, &prog_info, &prog_len);
618*053f45beSAndroid Build Coastguard Worker if (err)
619*053f45beSAndroid Build Coastguard Worker return err;
620*053f45beSAndroid Build Coastguard Worker
621*053f45beSAndroid Build Coastguard Worker num_maps = prog_info.nr_map_ids;
622*053f45beSAndroid Build Coastguard Worker
623*053f45beSAndroid Build Coastguard Worker map_ids = calloc(prog_info.nr_map_ids, sizeof(*map_ids));
624*053f45beSAndroid Build Coastguard Worker if (!map_ids)
625*053f45beSAndroid Build Coastguard Worker return -ENOMEM;
626*053f45beSAndroid Build Coastguard Worker
627*053f45beSAndroid Build Coastguard Worker memset(&prog_info, 0, prog_len);
628*053f45beSAndroid Build Coastguard Worker prog_info.nr_map_ids = num_maps;
629*053f45beSAndroid Build Coastguard Worker prog_info.map_ids = (__u64)(unsigned long)map_ids;
630*053f45beSAndroid Build Coastguard Worker
631*053f45beSAndroid Build Coastguard Worker err = bpf_obj_get_info_by_fd(ctx->prog_fd, &prog_info, &prog_len);
632*053f45beSAndroid Build Coastguard Worker if (err)
633*053f45beSAndroid Build Coastguard Worker goto out_map_ids;
634*053f45beSAndroid Build Coastguard Worker
635*053f45beSAndroid Build Coastguard Worker ctx->xsks_map_fd = -1;
636*053f45beSAndroid Build Coastguard Worker
637*053f45beSAndroid Build Coastguard Worker for (i = 0; i < prog_info.nr_map_ids; i++) {
638*053f45beSAndroid Build Coastguard Worker fd = bpf_map_get_fd_by_id(map_ids[i]);
639*053f45beSAndroid Build Coastguard Worker if (fd < 0)
640*053f45beSAndroid Build Coastguard Worker continue;
641*053f45beSAndroid Build Coastguard Worker
642*053f45beSAndroid Build Coastguard Worker memset(&map_info, 0, map_len);
643*053f45beSAndroid Build Coastguard Worker err = bpf_obj_get_info_by_fd(fd, &map_info, &map_len);
644*053f45beSAndroid Build Coastguard Worker if (err) {
645*053f45beSAndroid Build Coastguard Worker close(fd);
646*053f45beSAndroid Build Coastguard Worker continue;
647*053f45beSAndroid Build Coastguard Worker }
648*053f45beSAndroid Build Coastguard Worker
649*053f45beSAndroid Build Coastguard Worker if (!strncmp(map_info.name, "xsks_map", sizeof(map_info.name))) {
650*053f45beSAndroid Build Coastguard Worker ctx->xsks_map_fd = fd;
651*053f45beSAndroid Build Coastguard Worker break;
652*053f45beSAndroid Build Coastguard Worker }
653*053f45beSAndroid Build Coastguard Worker
654*053f45beSAndroid Build Coastguard Worker close(fd);
655*053f45beSAndroid Build Coastguard Worker }
656*053f45beSAndroid Build Coastguard Worker
657*053f45beSAndroid Build Coastguard Worker if (ctx->xsks_map_fd == -1)
658*053f45beSAndroid Build Coastguard Worker err = -ENOENT;
659*053f45beSAndroid Build Coastguard Worker
660*053f45beSAndroid Build Coastguard Worker out_map_ids:
661*053f45beSAndroid Build Coastguard Worker free(map_ids);
662*053f45beSAndroid Build Coastguard Worker return err;
663*053f45beSAndroid Build Coastguard Worker }
664*053f45beSAndroid Build Coastguard Worker
xsk_set_bpf_maps(struct xsk_socket * xsk)665*053f45beSAndroid Build Coastguard Worker static int xsk_set_bpf_maps(struct xsk_socket *xsk)
666*053f45beSAndroid Build Coastguard Worker {
667*053f45beSAndroid Build Coastguard Worker struct xsk_ctx *ctx = xsk->ctx;
668*053f45beSAndroid Build Coastguard Worker
669*053f45beSAndroid Build Coastguard Worker return bpf_map_update_elem(ctx->xsks_map_fd, &ctx->queue_id,
670*053f45beSAndroid Build Coastguard Worker &xsk->fd, 0);
671*053f45beSAndroid Build Coastguard Worker }
672*053f45beSAndroid Build Coastguard Worker
xsk_link_lookup(int ifindex,__u32 * prog_id,int * link_fd)673*053f45beSAndroid Build Coastguard Worker static int xsk_link_lookup(int ifindex, __u32 *prog_id, int *link_fd)
674*053f45beSAndroid Build Coastguard Worker {
675*053f45beSAndroid Build Coastguard Worker struct bpf_link_info link_info;
676*053f45beSAndroid Build Coastguard Worker __u32 link_len;
677*053f45beSAndroid Build Coastguard Worker __u32 id = 0;
678*053f45beSAndroid Build Coastguard Worker int err;
679*053f45beSAndroid Build Coastguard Worker int fd;
680*053f45beSAndroid Build Coastguard Worker
681*053f45beSAndroid Build Coastguard Worker while (true) {
682*053f45beSAndroid Build Coastguard Worker err = bpf_link_get_next_id(id, &id);
683*053f45beSAndroid Build Coastguard Worker if (err) {
684*053f45beSAndroid Build Coastguard Worker if (errno == ENOENT) {
685*053f45beSAndroid Build Coastguard Worker err = 0;
686*053f45beSAndroid Build Coastguard Worker break;
687*053f45beSAndroid Build Coastguard Worker }
688*053f45beSAndroid Build Coastguard Worker pr_warn("can't get next link: %s\n", strerror(errno));
689*053f45beSAndroid Build Coastguard Worker break;
690*053f45beSAndroid Build Coastguard Worker }
691*053f45beSAndroid Build Coastguard Worker
692*053f45beSAndroid Build Coastguard Worker fd = bpf_link_get_fd_by_id(id);
693*053f45beSAndroid Build Coastguard Worker if (fd < 0) {
694*053f45beSAndroid Build Coastguard Worker if (errno == ENOENT)
695*053f45beSAndroid Build Coastguard Worker continue;
696*053f45beSAndroid Build Coastguard Worker pr_warn("can't get link by id (%u): %s\n", id, strerror(errno));
697*053f45beSAndroid Build Coastguard Worker err = -errno;
698*053f45beSAndroid Build Coastguard Worker break;
699*053f45beSAndroid Build Coastguard Worker }
700*053f45beSAndroid Build Coastguard Worker
701*053f45beSAndroid Build Coastguard Worker link_len = sizeof(struct bpf_link_info);
702*053f45beSAndroid Build Coastguard Worker memset(&link_info, 0, link_len);
703*053f45beSAndroid Build Coastguard Worker err = bpf_obj_get_info_by_fd(fd, &link_info, &link_len);
704*053f45beSAndroid Build Coastguard Worker if (err) {
705*053f45beSAndroid Build Coastguard Worker pr_warn("can't get link info: %s\n", strerror(errno));
706*053f45beSAndroid Build Coastguard Worker close(fd);
707*053f45beSAndroid Build Coastguard Worker break;
708*053f45beSAndroid Build Coastguard Worker }
709*053f45beSAndroid Build Coastguard Worker if (link_info.type == BPF_LINK_TYPE_XDP) {
710*053f45beSAndroid Build Coastguard Worker if (link_info.xdp.ifindex == ifindex) {
711*053f45beSAndroid Build Coastguard Worker *link_fd = fd;
712*053f45beSAndroid Build Coastguard Worker if (prog_id)
713*053f45beSAndroid Build Coastguard Worker *prog_id = link_info.prog_id;
714*053f45beSAndroid Build Coastguard Worker break;
715*053f45beSAndroid Build Coastguard Worker }
716*053f45beSAndroid Build Coastguard Worker }
717*053f45beSAndroid Build Coastguard Worker close(fd);
718*053f45beSAndroid Build Coastguard Worker }
719*053f45beSAndroid Build Coastguard Worker
720*053f45beSAndroid Build Coastguard Worker return err;
721*053f45beSAndroid Build Coastguard Worker }
722*053f45beSAndroid Build Coastguard Worker
xsk_probe_bpf_link(void)723*053f45beSAndroid Build Coastguard Worker static bool xsk_probe_bpf_link(void)
724*053f45beSAndroid Build Coastguard Worker {
725*053f45beSAndroid Build Coastguard Worker LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = XDP_FLAGS_SKB_MODE);
726*053f45beSAndroid Build Coastguard Worker struct bpf_insn insns[2] = {
727*053f45beSAndroid Build Coastguard Worker BPF_MOV64_IMM(BPF_REG_0, XDP_PASS),
728*053f45beSAndroid Build Coastguard Worker BPF_EXIT_INSN()
729*053f45beSAndroid Build Coastguard Worker };
730*053f45beSAndroid Build Coastguard Worker int prog_fd, link_fd = -1, insn_cnt = ARRAY_SIZE(insns);
731*053f45beSAndroid Build Coastguard Worker int ifindex_lo = 1;
732*053f45beSAndroid Build Coastguard Worker bool ret = false;
733*053f45beSAndroid Build Coastguard Worker int err;
734*053f45beSAndroid Build Coastguard Worker
735*053f45beSAndroid Build Coastguard Worker err = xsk_link_lookup(ifindex_lo, NULL, &link_fd);
736*053f45beSAndroid Build Coastguard Worker if (err)
737*053f45beSAndroid Build Coastguard Worker return ret;
738*053f45beSAndroid Build Coastguard Worker
739*053f45beSAndroid Build Coastguard Worker if (link_fd >= 0)
740*053f45beSAndroid Build Coastguard Worker return true;
741*053f45beSAndroid Build Coastguard Worker
742*053f45beSAndroid Build Coastguard Worker prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL);
743*053f45beSAndroid Build Coastguard Worker if (prog_fd < 0)
744*053f45beSAndroid Build Coastguard Worker return ret;
745*053f45beSAndroid Build Coastguard Worker
746*053f45beSAndroid Build Coastguard Worker link_fd = bpf_link_create(prog_fd, ifindex_lo, BPF_XDP, &opts);
747*053f45beSAndroid Build Coastguard Worker close(prog_fd);
748*053f45beSAndroid Build Coastguard Worker
749*053f45beSAndroid Build Coastguard Worker if (link_fd >= 0) {
750*053f45beSAndroid Build Coastguard Worker ret = true;
751*053f45beSAndroid Build Coastguard Worker close(link_fd);
752*053f45beSAndroid Build Coastguard Worker }
753*053f45beSAndroid Build Coastguard Worker
754*053f45beSAndroid Build Coastguard Worker return ret;
755*053f45beSAndroid Build Coastguard Worker }
756*053f45beSAndroid Build Coastguard Worker
xsk_create_xsk_struct(int ifindex,struct xsk_socket * xsk)757*053f45beSAndroid Build Coastguard Worker static int xsk_create_xsk_struct(int ifindex, struct xsk_socket *xsk)
758*053f45beSAndroid Build Coastguard Worker {
759*053f45beSAndroid Build Coastguard Worker char ifname[IFNAMSIZ];
760*053f45beSAndroid Build Coastguard Worker struct xsk_ctx *ctx;
761*053f45beSAndroid Build Coastguard Worker char *interface;
762*053f45beSAndroid Build Coastguard Worker
763*053f45beSAndroid Build Coastguard Worker ctx = calloc(1, sizeof(*ctx));
764*053f45beSAndroid Build Coastguard Worker if (!ctx)
765*053f45beSAndroid Build Coastguard Worker return -ENOMEM;
766*053f45beSAndroid Build Coastguard Worker
767*053f45beSAndroid Build Coastguard Worker interface = if_indextoname(ifindex, &ifname[0]);
768*053f45beSAndroid Build Coastguard Worker if (!interface) {
769*053f45beSAndroid Build Coastguard Worker free(ctx);
770*053f45beSAndroid Build Coastguard Worker return -errno;
771*053f45beSAndroid Build Coastguard Worker }
772*053f45beSAndroid Build Coastguard Worker
773*053f45beSAndroid Build Coastguard Worker ctx->ifindex = ifindex;
774*053f45beSAndroid Build Coastguard Worker libbpf_strlcpy(ctx->ifname, ifname, IFNAMSIZ);
775*053f45beSAndroid Build Coastguard Worker
776*053f45beSAndroid Build Coastguard Worker xsk->ctx = ctx;
777*053f45beSAndroid Build Coastguard Worker xsk->ctx->has_bpf_link = xsk_probe_bpf_link();
778*053f45beSAndroid Build Coastguard Worker
779*053f45beSAndroid Build Coastguard Worker return 0;
780*053f45beSAndroid Build Coastguard Worker }
781*053f45beSAndroid Build Coastguard Worker
xsk_init_xdp_res(struct xsk_socket * xsk,int * xsks_map_fd)782*053f45beSAndroid Build Coastguard Worker static int xsk_init_xdp_res(struct xsk_socket *xsk,
783*053f45beSAndroid Build Coastguard Worker int *xsks_map_fd)
784*053f45beSAndroid Build Coastguard Worker {
785*053f45beSAndroid Build Coastguard Worker struct xsk_ctx *ctx = xsk->ctx;
786*053f45beSAndroid Build Coastguard Worker int err;
787*053f45beSAndroid Build Coastguard Worker
788*053f45beSAndroid Build Coastguard Worker err = xsk_create_bpf_maps(xsk);
789*053f45beSAndroid Build Coastguard Worker if (err)
790*053f45beSAndroid Build Coastguard Worker return err;
791*053f45beSAndroid Build Coastguard Worker
792*053f45beSAndroid Build Coastguard Worker err = xsk_load_xdp_prog(xsk);
793*053f45beSAndroid Build Coastguard Worker if (err)
794*053f45beSAndroid Build Coastguard Worker goto err_load_xdp_prog;
795*053f45beSAndroid Build Coastguard Worker
796*053f45beSAndroid Build Coastguard Worker if (ctx->has_bpf_link)
797*053f45beSAndroid Build Coastguard Worker err = xsk_create_bpf_link(xsk);
798*053f45beSAndroid Build Coastguard Worker else
799*053f45beSAndroid Build Coastguard Worker err = bpf_xdp_attach(xsk->ctx->ifindex, ctx->prog_fd,
800*053f45beSAndroid Build Coastguard Worker xsk->config.xdp_flags, NULL);
801*053f45beSAndroid Build Coastguard Worker
802*053f45beSAndroid Build Coastguard Worker if (err)
803*053f45beSAndroid Build Coastguard Worker goto err_attach_xdp_prog;
804*053f45beSAndroid Build Coastguard Worker
805*053f45beSAndroid Build Coastguard Worker if (!xsk->rx)
806*053f45beSAndroid Build Coastguard Worker return err;
807*053f45beSAndroid Build Coastguard Worker
808*053f45beSAndroid Build Coastguard Worker err = xsk_set_bpf_maps(xsk);
809*053f45beSAndroid Build Coastguard Worker if (err)
810*053f45beSAndroid Build Coastguard Worker goto err_set_bpf_maps;
811*053f45beSAndroid Build Coastguard Worker
812*053f45beSAndroid Build Coastguard Worker return err;
813*053f45beSAndroid Build Coastguard Worker
814*053f45beSAndroid Build Coastguard Worker err_set_bpf_maps:
815*053f45beSAndroid Build Coastguard Worker if (ctx->has_bpf_link)
816*053f45beSAndroid Build Coastguard Worker close(ctx->link_fd);
817*053f45beSAndroid Build Coastguard Worker else
818*053f45beSAndroid Build Coastguard Worker bpf_xdp_detach(ctx->ifindex, 0, NULL);
819*053f45beSAndroid Build Coastguard Worker err_attach_xdp_prog:
820*053f45beSAndroid Build Coastguard Worker close(ctx->prog_fd);
821*053f45beSAndroid Build Coastguard Worker err_load_xdp_prog:
822*053f45beSAndroid Build Coastguard Worker xsk_delete_bpf_maps(xsk);
823*053f45beSAndroid Build Coastguard Worker return err;
824*053f45beSAndroid Build Coastguard Worker }
825*053f45beSAndroid Build Coastguard Worker
xsk_lookup_xdp_res(struct xsk_socket * xsk,int * xsks_map_fd,int prog_id)826*053f45beSAndroid Build Coastguard Worker static int xsk_lookup_xdp_res(struct xsk_socket *xsk, int *xsks_map_fd, int prog_id)
827*053f45beSAndroid Build Coastguard Worker {
828*053f45beSAndroid Build Coastguard Worker struct xsk_ctx *ctx = xsk->ctx;
829*053f45beSAndroid Build Coastguard Worker int err;
830*053f45beSAndroid Build Coastguard Worker
831*053f45beSAndroid Build Coastguard Worker ctx->prog_fd = bpf_prog_get_fd_by_id(prog_id);
832*053f45beSAndroid Build Coastguard Worker if (ctx->prog_fd < 0) {
833*053f45beSAndroid Build Coastguard Worker err = -errno;
834*053f45beSAndroid Build Coastguard Worker goto err_prog_fd;
835*053f45beSAndroid Build Coastguard Worker }
836*053f45beSAndroid Build Coastguard Worker err = xsk_lookup_bpf_maps(xsk);
837*053f45beSAndroid Build Coastguard Worker if (err)
838*053f45beSAndroid Build Coastguard Worker goto err_lookup_maps;
839*053f45beSAndroid Build Coastguard Worker
840*053f45beSAndroid Build Coastguard Worker if (!xsk->rx)
841*053f45beSAndroid Build Coastguard Worker return err;
842*053f45beSAndroid Build Coastguard Worker
843*053f45beSAndroid Build Coastguard Worker err = xsk_set_bpf_maps(xsk);
844*053f45beSAndroid Build Coastguard Worker if (err)
845*053f45beSAndroid Build Coastguard Worker goto err_set_maps;
846*053f45beSAndroid Build Coastguard Worker
847*053f45beSAndroid Build Coastguard Worker return err;
848*053f45beSAndroid Build Coastguard Worker
849*053f45beSAndroid Build Coastguard Worker err_set_maps:
850*053f45beSAndroid Build Coastguard Worker close(ctx->xsks_map_fd);
851*053f45beSAndroid Build Coastguard Worker err_lookup_maps:
852*053f45beSAndroid Build Coastguard Worker close(ctx->prog_fd);
853*053f45beSAndroid Build Coastguard Worker err_prog_fd:
854*053f45beSAndroid Build Coastguard Worker if (ctx->has_bpf_link)
855*053f45beSAndroid Build Coastguard Worker close(ctx->link_fd);
856*053f45beSAndroid Build Coastguard Worker return err;
857*053f45beSAndroid Build Coastguard Worker }
858*053f45beSAndroid Build Coastguard Worker
__xsk_setup_xdp_prog(struct xsk_socket * _xdp,int * xsks_map_fd)859*053f45beSAndroid Build Coastguard Worker static int __xsk_setup_xdp_prog(struct xsk_socket *_xdp, int *xsks_map_fd)
860*053f45beSAndroid Build Coastguard Worker {
861*053f45beSAndroid Build Coastguard Worker struct xsk_socket *xsk = _xdp;
862*053f45beSAndroid Build Coastguard Worker struct xsk_ctx *ctx = xsk->ctx;
863*053f45beSAndroid Build Coastguard Worker __u32 prog_id = 0;
864*053f45beSAndroid Build Coastguard Worker int err;
865*053f45beSAndroid Build Coastguard Worker
866*053f45beSAndroid Build Coastguard Worker if (ctx->has_bpf_link)
867*053f45beSAndroid Build Coastguard Worker err = xsk_link_lookup(ctx->ifindex, &prog_id, &ctx->link_fd);
868*053f45beSAndroid Build Coastguard Worker else
869*053f45beSAndroid Build Coastguard Worker err = bpf_xdp_query_id(ctx->ifindex, xsk->config.xdp_flags, &prog_id);
870*053f45beSAndroid Build Coastguard Worker
871*053f45beSAndroid Build Coastguard Worker if (err)
872*053f45beSAndroid Build Coastguard Worker return err;
873*053f45beSAndroid Build Coastguard Worker
874*053f45beSAndroid Build Coastguard Worker err = !prog_id ? xsk_init_xdp_res(xsk, xsks_map_fd) :
875*053f45beSAndroid Build Coastguard Worker xsk_lookup_xdp_res(xsk, xsks_map_fd, prog_id);
876*053f45beSAndroid Build Coastguard Worker
877*053f45beSAndroid Build Coastguard Worker if (!err && xsks_map_fd)
878*053f45beSAndroid Build Coastguard Worker *xsks_map_fd = ctx->xsks_map_fd;
879*053f45beSAndroid Build Coastguard Worker
880*053f45beSAndroid Build Coastguard Worker return err;
881*053f45beSAndroid Build Coastguard Worker }
882*053f45beSAndroid Build Coastguard Worker
xsk_setup_xdp_prog_xsk(struct xsk_socket * xsk,int * xsks_map_fd)883*053f45beSAndroid Build Coastguard Worker int xsk_setup_xdp_prog_xsk(struct xsk_socket *xsk, int *xsks_map_fd)
884*053f45beSAndroid Build Coastguard Worker {
885*053f45beSAndroid Build Coastguard Worker return __xsk_setup_xdp_prog(xsk, xsks_map_fd);
886*053f45beSAndroid Build Coastguard Worker }
887*053f45beSAndroid Build Coastguard Worker
xsk_get_ctx(struct xsk_umem * umem,int ifindex,__u32 queue_id)888*053f45beSAndroid Build Coastguard Worker static struct xsk_ctx *xsk_get_ctx(struct xsk_umem *umem, int ifindex,
889*053f45beSAndroid Build Coastguard Worker __u32 queue_id)
890*053f45beSAndroid Build Coastguard Worker {
891*053f45beSAndroid Build Coastguard Worker struct xsk_ctx *ctx;
892*053f45beSAndroid Build Coastguard Worker
893*053f45beSAndroid Build Coastguard Worker if (list_empty(&umem->ctx_list))
894*053f45beSAndroid Build Coastguard Worker return NULL;
895*053f45beSAndroid Build Coastguard Worker
896*053f45beSAndroid Build Coastguard Worker list_for_each_entry(ctx, &umem->ctx_list, list) {
897*053f45beSAndroid Build Coastguard Worker if (ctx->ifindex == ifindex && ctx->queue_id == queue_id) {
898*053f45beSAndroid Build Coastguard Worker ctx->refcount++;
899*053f45beSAndroid Build Coastguard Worker return ctx;
900*053f45beSAndroid Build Coastguard Worker }
901*053f45beSAndroid Build Coastguard Worker }
902*053f45beSAndroid Build Coastguard Worker
903*053f45beSAndroid Build Coastguard Worker return NULL;
904*053f45beSAndroid Build Coastguard Worker }
905*053f45beSAndroid Build Coastguard Worker
xsk_put_ctx(struct xsk_ctx * ctx,bool unmap)906*053f45beSAndroid Build Coastguard Worker static void xsk_put_ctx(struct xsk_ctx *ctx, bool unmap)
907*053f45beSAndroid Build Coastguard Worker {
908*053f45beSAndroid Build Coastguard Worker struct xsk_umem *umem = ctx->umem;
909*053f45beSAndroid Build Coastguard Worker struct xdp_mmap_offsets off;
910*053f45beSAndroid Build Coastguard Worker int err;
911*053f45beSAndroid Build Coastguard Worker
912*053f45beSAndroid Build Coastguard Worker if (--ctx->refcount)
913*053f45beSAndroid Build Coastguard Worker return;
914*053f45beSAndroid Build Coastguard Worker
915*053f45beSAndroid Build Coastguard Worker if (!unmap)
916*053f45beSAndroid Build Coastguard Worker goto out_free;
917*053f45beSAndroid Build Coastguard Worker
918*053f45beSAndroid Build Coastguard Worker err = xsk_get_mmap_offsets(umem->fd, &off);
919*053f45beSAndroid Build Coastguard Worker if (err)
920*053f45beSAndroid Build Coastguard Worker goto out_free;
921*053f45beSAndroid Build Coastguard Worker
922*053f45beSAndroid Build Coastguard Worker munmap(ctx->fill->ring - off.fr.desc, off.fr.desc + umem->config.fill_size *
923*053f45beSAndroid Build Coastguard Worker sizeof(__u64));
924*053f45beSAndroid Build Coastguard Worker munmap(ctx->comp->ring - off.cr.desc, off.cr.desc + umem->config.comp_size *
925*053f45beSAndroid Build Coastguard Worker sizeof(__u64));
926*053f45beSAndroid Build Coastguard Worker
927*053f45beSAndroid Build Coastguard Worker out_free:
928*053f45beSAndroid Build Coastguard Worker list_del(&ctx->list);
929*053f45beSAndroid Build Coastguard Worker free(ctx);
930*053f45beSAndroid Build Coastguard Worker }
931*053f45beSAndroid Build Coastguard Worker
xsk_create_ctx(struct xsk_socket * xsk,struct xsk_umem * umem,int ifindex,const char * ifname,__u32 queue_id,struct xsk_ring_prod * fill,struct xsk_ring_cons * comp)932*053f45beSAndroid Build Coastguard Worker static struct xsk_ctx *xsk_create_ctx(struct xsk_socket *xsk,
933*053f45beSAndroid Build Coastguard Worker struct xsk_umem *umem, int ifindex,
934*053f45beSAndroid Build Coastguard Worker const char *ifname, __u32 queue_id,
935*053f45beSAndroid Build Coastguard Worker struct xsk_ring_prod *fill,
936*053f45beSAndroid Build Coastguard Worker struct xsk_ring_cons *comp)
937*053f45beSAndroid Build Coastguard Worker {
938*053f45beSAndroid Build Coastguard Worker struct xsk_ctx *ctx;
939*053f45beSAndroid Build Coastguard Worker int err;
940*053f45beSAndroid Build Coastguard Worker
941*053f45beSAndroid Build Coastguard Worker ctx = calloc(1, sizeof(*ctx));
942*053f45beSAndroid Build Coastguard Worker if (!ctx)
943*053f45beSAndroid Build Coastguard Worker return NULL;
944*053f45beSAndroid Build Coastguard Worker
945*053f45beSAndroid Build Coastguard Worker if (!umem->fill_save) {
946*053f45beSAndroid Build Coastguard Worker err = xsk_create_umem_rings(umem, xsk->fd, fill, comp);
947*053f45beSAndroid Build Coastguard Worker if (err) {
948*053f45beSAndroid Build Coastguard Worker free(ctx);
949*053f45beSAndroid Build Coastguard Worker return NULL;
950*053f45beSAndroid Build Coastguard Worker }
951*053f45beSAndroid Build Coastguard Worker } else if (umem->fill_save != fill || umem->comp_save != comp) {
952*053f45beSAndroid Build Coastguard Worker /* Copy over rings to new structs. */
953*053f45beSAndroid Build Coastguard Worker memcpy(fill, umem->fill_save, sizeof(*fill));
954*053f45beSAndroid Build Coastguard Worker memcpy(comp, umem->comp_save, sizeof(*comp));
955*053f45beSAndroid Build Coastguard Worker }
956*053f45beSAndroid Build Coastguard Worker
957*053f45beSAndroid Build Coastguard Worker ctx->ifindex = ifindex;
958*053f45beSAndroid Build Coastguard Worker ctx->refcount = 1;
959*053f45beSAndroid Build Coastguard Worker ctx->umem = umem;
960*053f45beSAndroid Build Coastguard Worker ctx->queue_id = queue_id;
961*053f45beSAndroid Build Coastguard Worker libbpf_strlcpy(ctx->ifname, ifname, IFNAMSIZ);
962*053f45beSAndroid Build Coastguard Worker
963*053f45beSAndroid Build Coastguard Worker ctx->fill = fill;
964*053f45beSAndroid Build Coastguard Worker ctx->comp = comp;
965*053f45beSAndroid Build Coastguard Worker list_add(&ctx->list, &umem->ctx_list);
966*053f45beSAndroid Build Coastguard Worker ctx->has_bpf_link = xsk_probe_bpf_link();
967*053f45beSAndroid Build Coastguard Worker return ctx;
968*053f45beSAndroid Build Coastguard Worker }
969*053f45beSAndroid Build Coastguard Worker
xsk_destroy_xsk_struct(struct xsk_socket * xsk)970*053f45beSAndroid Build Coastguard Worker static void xsk_destroy_xsk_struct(struct xsk_socket *xsk)
971*053f45beSAndroid Build Coastguard Worker {
972*053f45beSAndroid Build Coastguard Worker free(xsk->ctx);
973*053f45beSAndroid Build Coastguard Worker free(xsk);
974*053f45beSAndroid Build Coastguard Worker }
975*053f45beSAndroid Build Coastguard Worker
xsk_socket__update_xskmap(struct xsk_socket * xsk,int fd)976*053f45beSAndroid Build Coastguard Worker int xsk_socket__update_xskmap(struct xsk_socket *xsk, int fd)
977*053f45beSAndroid Build Coastguard Worker {
978*053f45beSAndroid Build Coastguard Worker xsk->ctx->xsks_map_fd = fd;
979*053f45beSAndroid Build Coastguard Worker return xsk_set_bpf_maps(xsk);
980*053f45beSAndroid Build Coastguard Worker }
981*053f45beSAndroid Build Coastguard Worker
xsk_setup_xdp_prog(int ifindex,int * xsks_map_fd)982*053f45beSAndroid Build Coastguard Worker int xsk_setup_xdp_prog(int ifindex, int *xsks_map_fd)
983*053f45beSAndroid Build Coastguard Worker {
984*053f45beSAndroid Build Coastguard Worker struct xsk_socket *xsk;
985*053f45beSAndroid Build Coastguard Worker int res;
986*053f45beSAndroid Build Coastguard Worker
987*053f45beSAndroid Build Coastguard Worker xsk = calloc(1, sizeof(*xsk));
988*053f45beSAndroid Build Coastguard Worker if (!xsk)
989*053f45beSAndroid Build Coastguard Worker return -ENOMEM;
990*053f45beSAndroid Build Coastguard Worker
991*053f45beSAndroid Build Coastguard Worker res = xsk_create_xsk_struct(ifindex, xsk);
992*053f45beSAndroid Build Coastguard Worker if (res) {
993*053f45beSAndroid Build Coastguard Worker free(xsk);
994*053f45beSAndroid Build Coastguard Worker return -EINVAL;
995*053f45beSAndroid Build Coastguard Worker }
996*053f45beSAndroid Build Coastguard Worker
997*053f45beSAndroid Build Coastguard Worker res = __xsk_setup_xdp_prog(xsk, xsks_map_fd);
998*053f45beSAndroid Build Coastguard Worker
999*053f45beSAndroid Build Coastguard Worker xsk_destroy_xsk_struct(xsk);
1000*053f45beSAndroid Build Coastguard Worker
1001*053f45beSAndroid Build Coastguard Worker return res;
1002*053f45beSAndroid Build Coastguard Worker }
1003*053f45beSAndroid Build Coastguard Worker
xsk_socket__create_shared(struct xsk_socket ** xsk_ptr,const char * ifname,__u32 queue_id,struct xsk_umem * umem,struct xsk_ring_cons * rx,struct xsk_ring_prod * tx,struct xsk_ring_prod * fill,struct xsk_ring_cons * comp,const struct xsk_socket_config * usr_config)1004*053f45beSAndroid Build Coastguard Worker int xsk_socket__create_shared(struct xsk_socket **xsk_ptr,
1005*053f45beSAndroid Build Coastguard Worker const char *ifname,
1006*053f45beSAndroid Build Coastguard Worker __u32 queue_id, struct xsk_umem *umem,
1007*053f45beSAndroid Build Coastguard Worker struct xsk_ring_cons *rx,
1008*053f45beSAndroid Build Coastguard Worker struct xsk_ring_prod *tx,
1009*053f45beSAndroid Build Coastguard Worker struct xsk_ring_prod *fill,
1010*053f45beSAndroid Build Coastguard Worker struct xsk_ring_cons *comp,
1011*053f45beSAndroid Build Coastguard Worker const struct xsk_socket_config *usr_config)
1012*053f45beSAndroid Build Coastguard Worker {
1013*053f45beSAndroid Build Coastguard Worker bool unmap, rx_setup_done = false, tx_setup_done = false;
1014*053f45beSAndroid Build Coastguard Worker void *rx_map = NULL, *tx_map = NULL;
1015*053f45beSAndroid Build Coastguard Worker struct sockaddr_xdp sxdp = {};
1016*053f45beSAndroid Build Coastguard Worker struct xdp_mmap_offsets off;
1017*053f45beSAndroid Build Coastguard Worker struct xsk_socket *xsk;
1018*053f45beSAndroid Build Coastguard Worker struct xsk_ctx *ctx;
1019*053f45beSAndroid Build Coastguard Worker int err, ifindex;
1020*053f45beSAndroid Build Coastguard Worker
1021*053f45beSAndroid Build Coastguard Worker if (!umem || !xsk_ptr || !(rx || tx))
1022*053f45beSAndroid Build Coastguard Worker return -EFAULT;
1023*053f45beSAndroid Build Coastguard Worker
1024*053f45beSAndroid Build Coastguard Worker unmap = umem->fill_save != fill;
1025*053f45beSAndroid Build Coastguard Worker
1026*053f45beSAndroid Build Coastguard Worker xsk = calloc(1, sizeof(*xsk));
1027*053f45beSAndroid Build Coastguard Worker if (!xsk)
1028*053f45beSAndroid Build Coastguard Worker return -ENOMEM;
1029*053f45beSAndroid Build Coastguard Worker
1030*053f45beSAndroid Build Coastguard Worker err = xsk_set_xdp_socket_config(&xsk->config, usr_config);
1031*053f45beSAndroid Build Coastguard Worker if (err)
1032*053f45beSAndroid Build Coastguard Worker goto out_xsk_alloc;
1033*053f45beSAndroid Build Coastguard Worker
1034*053f45beSAndroid Build Coastguard Worker xsk->outstanding_tx = 0;
1035*053f45beSAndroid Build Coastguard Worker ifindex = if_nametoindex(ifname);
1036*053f45beSAndroid Build Coastguard Worker if (!ifindex) {
1037*053f45beSAndroid Build Coastguard Worker err = -errno;
1038*053f45beSAndroid Build Coastguard Worker goto out_xsk_alloc;
1039*053f45beSAndroid Build Coastguard Worker }
1040*053f45beSAndroid Build Coastguard Worker
1041*053f45beSAndroid Build Coastguard Worker if (umem->refcount++ > 0) {
1042*053f45beSAndroid Build Coastguard Worker xsk->fd = socket(AF_XDP, SOCK_RAW | SOCK_CLOEXEC, 0);
1043*053f45beSAndroid Build Coastguard Worker if (xsk->fd < 0) {
1044*053f45beSAndroid Build Coastguard Worker err = -errno;
1045*053f45beSAndroid Build Coastguard Worker goto out_xsk_alloc;
1046*053f45beSAndroid Build Coastguard Worker }
1047*053f45beSAndroid Build Coastguard Worker } else {
1048*053f45beSAndroid Build Coastguard Worker xsk->fd = umem->fd;
1049*053f45beSAndroid Build Coastguard Worker rx_setup_done = umem->rx_ring_setup_done;
1050*053f45beSAndroid Build Coastguard Worker tx_setup_done = umem->tx_ring_setup_done;
1051*053f45beSAndroid Build Coastguard Worker }
1052*053f45beSAndroid Build Coastguard Worker
1053*053f45beSAndroid Build Coastguard Worker ctx = xsk_get_ctx(umem, ifindex, queue_id);
1054*053f45beSAndroid Build Coastguard Worker if (!ctx) {
1055*053f45beSAndroid Build Coastguard Worker if (!fill || !comp) {
1056*053f45beSAndroid Build Coastguard Worker err = -EFAULT;
1057*053f45beSAndroid Build Coastguard Worker goto out_socket;
1058*053f45beSAndroid Build Coastguard Worker }
1059*053f45beSAndroid Build Coastguard Worker
1060*053f45beSAndroid Build Coastguard Worker ctx = xsk_create_ctx(xsk, umem, ifindex, ifname, queue_id,
1061*053f45beSAndroid Build Coastguard Worker fill, comp);
1062*053f45beSAndroid Build Coastguard Worker if (!ctx) {
1063*053f45beSAndroid Build Coastguard Worker err = -ENOMEM;
1064*053f45beSAndroid Build Coastguard Worker goto out_socket;
1065*053f45beSAndroid Build Coastguard Worker }
1066*053f45beSAndroid Build Coastguard Worker }
1067*053f45beSAndroid Build Coastguard Worker xsk->ctx = ctx;
1068*053f45beSAndroid Build Coastguard Worker
1069*053f45beSAndroid Build Coastguard Worker if (rx && !rx_setup_done) {
1070*053f45beSAndroid Build Coastguard Worker err = setsockopt(xsk->fd, SOL_XDP, XDP_RX_RING,
1071*053f45beSAndroid Build Coastguard Worker &xsk->config.rx_size,
1072*053f45beSAndroid Build Coastguard Worker sizeof(xsk->config.rx_size));
1073*053f45beSAndroid Build Coastguard Worker if (err) {
1074*053f45beSAndroid Build Coastguard Worker err = -errno;
1075*053f45beSAndroid Build Coastguard Worker goto out_put_ctx;
1076*053f45beSAndroid Build Coastguard Worker }
1077*053f45beSAndroid Build Coastguard Worker if (xsk->fd == umem->fd)
1078*053f45beSAndroid Build Coastguard Worker umem->rx_ring_setup_done = true;
1079*053f45beSAndroid Build Coastguard Worker }
1080*053f45beSAndroid Build Coastguard Worker if (tx && !tx_setup_done) {
1081*053f45beSAndroid Build Coastguard Worker err = setsockopt(xsk->fd, SOL_XDP, XDP_TX_RING,
1082*053f45beSAndroid Build Coastguard Worker &xsk->config.tx_size,
1083*053f45beSAndroid Build Coastguard Worker sizeof(xsk->config.tx_size));
1084*053f45beSAndroid Build Coastguard Worker if (err) {
1085*053f45beSAndroid Build Coastguard Worker err = -errno;
1086*053f45beSAndroid Build Coastguard Worker goto out_put_ctx;
1087*053f45beSAndroid Build Coastguard Worker }
1088*053f45beSAndroid Build Coastguard Worker if (xsk->fd == umem->fd)
1089*053f45beSAndroid Build Coastguard Worker umem->tx_ring_setup_done = true;
1090*053f45beSAndroid Build Coastguard Worker }
1091*053f45beSAndroid Build Coastguard Worker
1092*053f45beSAndroid Build Coastguard Worker err = xsk_get_mmap_offsets(xsk->fd, &off);
1093*053f45beSAndroid Build Coastguard Worker if (err) {
1094*053f45beSAndroid Build Coastguard Worker err = -errno;
1095*053f45beSAndroid Build Coastguard Worker goto out_put_ctx;
1096*053f45beSAndroid Build Coastguard Worker }
1097*053f45beSAndroid Build Coastguard Worker
1098*053f45beSAndroid Build Coastguard Worker if (rx) {
1099*053f45beSAndroid Build Coastguard Worker rx_map = mmap(NULL, off.rx.desc +
1100*053f45beSAndroid Build Coastguard Worker xsk->config.rx_size * sizeof(struct xdp_desc),
1101*053f45beSAndroid Build Coastguard Worker PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
1102*053f45beSAndroid Build Coastguard Worker xsk->fd, XDP_PGOFF_RX_RING);
1103*053f45beSAndroid Build Coastguard Worker if (rx_map == MAP_FAILED) {
1104*053f45beSAndroid Build Coastguard Worker err = -errno;
1105*053f45beSAndroid Build Coastguard Worker goto out_put_ctx;
1106*053f45beSAndroid Build Coastguard Worker }
1107*053f45beSAndroid Build Coastguard Worker
1108*053f45beSAndroid Build Coastguard Worker rx->mask = xsk->config.rx_size - 1;
1109*053f45beSAndroid Build Coastguard Worker rx->size = xsk->config.rx_size;
1110*053f45beSAndroid Build Coastguard Worker rx->producer = rx_map + off.rx.producer;
1111*053f45beSAndroid Build Coastguard Worker rx->consumer = rx_map + off.rx.consumer;
1112*053f45beSAndroid Build Coastguard Worker rx->flags = rx_map + off.rx.flags;
1113*053f45beSAndroid Build Coastguard Worker rx->ring = rx_map + off.rx.desc;
1114*053f45beSAndroid Build Coastguard Worker rx->cached_prod = *rx->producer;
1115*053f45beSAndroid Build Coastguard Worker rx->cached_cons = *rx->consumer;
1116*053f45beSAndroid Build Coastguard Worker }
1117*053f45beSAndroid Build Coastguard Worker xsk->rx = rx;
1118*053f45beSAndroid Build Coastguard Worker
1119*053f45beSAndroid Build Coastguard Worker if (tx) {
1120*053f45beSAndroid Build Coastguard Worker tx_map = mmap(NULL, off.tx.desc +
1121*053f45beSAndroid Build Coastguard Worker xsk->config.tx_size * sizeof(struct xdp_desc),
1122*053f45beSAndroid Build Coastguard Worker PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
1123*053f45beSAndroid Build Coastguard Worker xsk->fd, XDP_PGOFF_TX_RING);
1124*053f45beSAndroid Build Coastguard Worker if (tx_map == MAP_FAILED) {
1125*053f45beSAndroid Build Coastguard Worker err = -errno;
1126*053f45beSAndroid Build Coastguard Worker goto out_mmap_rx;
1127*053f45beSAndroid Build Coastguard Worker }
1128*053f45beSAndroid Build Coastguard Worker
1129*053f45beSAndroid Build Coastguard Worker tx->mask = xsk->config.tx_size - 1;
1130*053f45beSAndroid Build Coastguard Worker tx->size = xsk->config.tx_size;
1131*053f45beSAndroid Build Coastguard Worker tx->producer = tx_map + off.tx.producer;
1132*053f45beSAndroid Build Coastguard Worker tx->consumer = tx_map + off.tx.consumer;
1133*053f45beSAndroid Build Coastguard Worker tx->flags = tx_map + off.tx.flags;
1134*053f45beSAndroid Build Coastguard Worker tx->ring = tx_map + off.tx.desc;
1135*053f45beSAndroid Build Coastguard Worker tx->cached_prod = *tx->producer;
1136*053f45beSAndroid Build Coastguard Worker /* cached_cons is r->size bigger than the real consumer pointer
1137*053f45beSAndroid Build Coastguard Worker * See xsk_prod_nb_free
1138*053f45beSAndroid Build Coastguard Worker */
1139*053f45beSAndroid Build Coastguard Worker tx->cached_cons = *tx->consumer + xsk->config.tx_size;
1140*053f45beSAndroid Build Coastguard Worker }
1141*053f45beSAndroid Build Coastguard Worker xsk->tx = tx;
1142*053f45beSAndroid Build Coastguard Worker
1143*053f45beSAndroid Build Coastguard Worker sxdp.sxdp_family = PF_XDP;
1144*053f45beSAndroid Build Coastguard Worker sxdp.sxdp_ifindex = ctx->ifindex;
1145*053f45beSAndroid Build Coastguard Worker sxdp.sxdp_queue_id = ctx->queue_id;
1146*053f45beSAndroid Build Coastguard Worker if (umem->refcount > 1) {
1147*053f45beSAndroid Build Coastguard Worker sxdp.sxdp_flags |= XDP_SHARED_UMEM;
1148*053f45beSAndroid Build Coastguard Worker sxdp.sxdp_shared_umem_fd = umem->fd;
1149*053f45beSAndroid Build Coastguard Worker } else {
1150*053f45beSAndroid Build Coastguard Worker sxdp.sxdp_flags = xsk->config.bind_flags;
1151*053f45beSAndroid Build Coastguard Worker }
1152*053f45beSAndroid Build Coastguard Worker
1153*053f45beSAndroid Build Coastguard Worker err = bind(xsk->fd, (struct sockaddr *)&sxdp, sizeof(sxdp));
1154*053f45beSAndroid Build Coastguard Worker if (err) {
1155*053f45beSAndroid Build Coastguard Worker err = -errno;
1156*053f45beSAndroid Build Coastguard Worker goto out_mmap_tx;
1157*053f45beSAndroid Build Coastguard Worker }
1158*053f45beSAndroid Build Coastguard Worker
1159*053f45beSAndroid Build Coastguard Worker if (!(xsk->config.libbpf_flags & XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD)) {
1160*053f45beSAndroid Build Coastguard Worker err = __xsk_setup_xdp_prog(xsk, NULL);
1161*053f45beSAndroid Build Coastguard Worker if (err)
1162*053f45beSAndroid Build Coastguard Worker goto out_mmap_tx;
1163*053f45beSAndroid Build Coastguard Worker }
1164*053f45beSAndroid Build Coastguard Worker
1165*053f45beSAndroid Build Coastguard Worker *xsk_ptr = xsk;
1166*053f45beSAndroid Build Coastguard Worker umem->fill_save = NULL;
1167*053f45beSAndroid Build Coastguard Worker umem->comp_save = NULL;
1168*053f45beSAndroid Build Coastguard Worker return 0;
1169*053f45beSAndroid Build Coastguard Worker
1170*053f45beSAndroid Build Coastguard Worker out_mmap_tx:
1171*053f45beSAndroid Build Coastguard Worker if (tx)
1172*053f45beSAndroid Build Coastguard Worker munmap(tx_map, off.tx.desc +
1173*053f45beSAndroid Build Coastguard Worker xsk->config.tx_size * sizeof(struct xdp_desc));
1174*053f45beSAndroid Build Coastguard Worker out_mmap_rx:
1175*053f45beSAndroid Build Coastguard Worker if (rx)
1176*053f45beSAndroid Build Coastguard Worker munmap(rx_map, off.rx.desc +
1177*053f45beSAndroid Build Coastguard Worker xsk->config.rx_size * sizeof(struct xdp_desc));
1178*053f45beSAndroid Build Coastguard Worker out_put_ctx:
1179*053f45beSAndroid Build Coastguard Worker xsk_put_ctx(ctx, unmap);
1180*053f45beSAndroid Build Coastguard Worker out_socket:
1181*053f45beSAndroid Build Coastguard Worker if (--umem->refcount)
1182*053f45beSAndroid Build Coastguard Worker close(xsk->fd);
1183*053f45beSAndroid Build Coastguard Worker out_xsk_alloc:
1184*053f45beSAndroid Build Coastguard Worker free(xsk);
1185*053f45beSAndroid Build Coastguard Worker return err;
1186*053f45beSAndroid Build Coastguard Worker }
1187*053f45beSAndroid Build Coastguard Worker
xsk_socket__create(struct xsk_socket ** xsk_ptr,const char * ifname,__u32 queue_id,struct xsk_umem * umem,struct xsk_ring_cons * rx,struct xsk_ring_prod * tx,const struct xsk_socket_config * usr_config)1188*053f45beSAndroid Build Coastguard Worker int xsk_socket__create(struct xsk_socket **xsk_ptr, const char *ifname,
1189*053f45beSAndroid Build Coastguard Worker __u32 queue_id, struct xsk_umem *umem,
1190*053f45beSAndroid Build Coastguard Worker struct xsk_ring_cons *rx, struct xsk_ring_prod *tx,
1191*053f45beSAndroid Build Coastguard Worker const struct xsk_socket_config *usr_config)
1192*053f45beSAndroid Build Coastguard Worker {
1193*053f45beSAndroid Build Coastguard Worker if (!umem)
1194*053f45beSAndroid Build Coastguard Worker return -EFAULT;
1195*053f45beSAndroid Build Coastguard Worker
1196*053f45beSAndroid Build Coastguard Worker return xsk_socket__create_shared(xsk_ptr, ifname, queue_id, umem,
1197*053f45beSAndroid Build Coastguard Worker rx, tx, umem->fill_save,
1198*053f45beSAndroid Build Coastguard Worker umem->comp_save, usr_config);
1199*053f45beSAndroid Build Coastguard Worker }
1200*053f45beSAndroid Build Coastguard Worker
xsk_umem__delete(struct xsk_umem * umem)1201*053f45beSAndroid Build Coastguard Worker int xsk_umem__delete(struct xsk_umem *umem)
1202*053f45beSAndroid Build Coastguard Worker {
1203*053f45beSAndroid Build Coastguard Worker struct xdp_mmap_offsets off;
1204*053f45beSAndroid Build Coastguard Worker int err;
1205*053f45beSAndroid Build Coastguard Worker
1206*053f45beSAndroid Build Coastguard Worker if (!umem)
1207*053f45beSAndroid Build Coastguard Worker return 0;
1208*053f45beSAndroid Build Coastguard Worker
1209*053f45beSAndroid Build Coastguard Worker if (umem->refcount)
1210*053f45beSAndroid Build Coastguard Worker return -EBUSY;
1211*053f45beSAndroid Build Coastguard Worker
1212*053f45beSAndroid Build Coastguard Worker err = xsk_get_mmap_offsets(umem->fd, &off);
1213*053f45beSAndroid Build Coastguard Worker if (!err && umem->fill_save && umem->comp_save) {
1214*053f45beSAndroid Build Coastguard Worker munmap(umem->fill_save->ring - off.fr.desc,
1215*053f45beSAndroid Build Coastguard Worker off.fr.desc + umem->config.fill_size * sizeof(__u64));
1216*053f45beSAndroid Build Coastguard Worker munmap(umem->comp_save->ring - off.cr.desc,
1217*053f45beSAndroid Build Coastguard Worker off.cr.desc + umem->config.comp_size * sizeof(__u64));
1218*053f45beSAndroid Build Coastguard Worker }
1219*053f45beSAndroid Build Coastguard Worker
1220*053f45beSAndroid Build Coastguard Worker close(umem->fd);
1221*053f45beSAndroid Build Coastguard Worker free(umem);
1222*053f45beSAndroid Build Coastguard Worker
1223*053f45beSAndroid Build Coastguard Worker return 0;
1224*053f45beSAndroid Build Coastguard Worker }
1225*053f45beSAndroid Build Coastguard Worker
xsk_socket__delete(struct xsk_socket * xsk)1226*053f45beSAndroid Build Coastguard Worker void xsk_socket__delete(struct xsk_socket *xsk)
1227*053f45beSAndroid Build Coastguard Worker {
1228*053f45beSAndroid Build Coastguard Worker size_t desc_sz = sizeof(struct xdp_desc);
1229*053f45beSAndroid Build Coastguard Worker struct xdp_mmap_offsets off;
1230*053f45beSAndroid Build Coastguard Worker struct xsk_umem *umem;
1231*053f45beSAndroid Build Coastguard Worker struct xsk_ctx *ctx;
1232*053f45beSAndroid Build Coastguard Worker int err;
1233*053f45beSAndroid Build Coastguard Worker
1234*053f45beSAndroid Build Coastguard Worker if (!xsk)
1235*053f45beSAndroid Build Coastguard Worker return;
1236*053f45beSAndroid Build Coastguard Worker
1237*053f45beSAndroid Build Coastguard Worker ctx = xsk->ctx;
1238*053f45beSAndroid Build Coastguard Worker umem = ctx->umem;
1239*053f45beSAndroid Build Coastguard Worker
1240*053f45beSAndroid Build Coastguard Worker if (ctx->refcount == 1) {
1241*053f45beSAndroid Build Coastguard Worker xsk_delete_bpf_maps(xsk);
1242*053f45beSAndroid Build Coastguard Worker close(ctx->prog_fd);
1243*053f45beSAndroid Build Coastguard Worker if (ctx->has_bpf_link)
1244*053f45beSAndroid Build Coastguard Worker close(ctx->link_fd);
1245*053f45beSAndroid Build Coastguard Worker }
1246*053f45beSAndroid Build Coastguard Worker
1247*053f45beSAndroid Build Coastguard Worker xsk_put_ctx(ctx, true);
1248*053f45beSAndroid Build Coastguard Worker
1249*053f45beSAndroid Build Coastguard Worker err = xsk_get_mmap_offsets(xsk->fd, &off);
1250*053f45beSAndroid Build Coastguard Worker if (!err) {
1251*053f45beSAndroid Build Coastguard Worker if (xsk->rx) {
1252*053f45beSAndroid Build Coastguard Worker munmap(xsk->rx->ring - off.rx.desc,
1253*053f45beSAndroid Build Coastguard Worker off.rx.desc + xsk->config.rx_size * desc_sz);
1254*053f45beSAndroid Build Coastguard Worker }
1255*053f45beSAndroid Build Coastguard Worker if (xsk->tx) {
1256*053f45beSAndroid Build Coastguard Worker munmap(xsk->tx->ring - off.tx.desc,
1257*053f45beSAndroid Build Coastguard Worker off.tx.desc + xsk->config.tx_size * desc_sz);
1258*053f45beSAndroid Build Coastguard Worker }
1259*053f45beSAndroid Build Coastguard Worker }
1260*053f45beSAndroid Build Coastguard Worker
1261*053f45beSAndroid Build Coastguard Worker umem->refcount--;
1262*053f45beSAndroid Build Coastguard Worker /* Do not close an fd that also has an associated umem connected
1263*053f45beSAndroid Build Coastguard Worker * to it.
1264*053f45beSAndroid Build Coastguard Worker */
1265*053f45beSAndroid Build Coastguard Worker if (xsk->fd != umem->fd)
1266*053f45beSAndroid Build Coastguard Worker close(xsk->fd);
1267*053f45beSAndroid Build Coastguard Worker free(xsk);
1268*053f45beSAndroid Build Coastguard Worker }
1269