1 // SPDX-License-Identifier: GPL-2.0-only
2
3 #include <linux/netdevice.h>
4 #include <linux/notifier.h>
5 #include <linux/rtnetlink.h>
6 #include <net/busy_poll.h>
7 #include <net/net_namespace.h>
8 #include <net/netdev_queues.h>
9 #include <net/netdev_rx_queue.h>
10 #include <net/sock.h>
11 #include <net/xdp.h>
12 #include <net/xdp_sock.h>
13
14 #include "dev.h"
15 #include "devmem.h"
16 #include "netdev-genl-gen.h"
17
18 struct netdev_nl_dump_ctx {
19 unsigned long ifindex;
20 unsigned int rxq_idx;
21 unsigned int txq_idx;
22 unsigned int napi_id;
23 };
24
netdev_dump_ctx(struct netlink_callback * cb)25 static struct netdev_nl_dump_ctx *netdev_dump_ctx(struct netlink_callback *cb)
26 {
27 NL_ASSERT_CTX_FITS(struct netdev_nl_dump_ctx);
28
29 return (struct netdev_nl_dump_ctx *)cb->ctx;
30 }
31
32 static int
netdev_nl_dev_fill(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info)33 netdev_nl_dev_fill(struct net_device *netdev, struct sk_buff *rsp,
34 const struct genl_info *info)
35 {
36 u64 xsk_features = 0;
37 u64 xdp_rx_meta = 0;
38 void *hdr;
39
40 hdr = genlmsg_iput(rsp, info);
41 if (!hdr)
42 return -EMSGSIZE;
43
44 #define XDP_METADATA_KFUNC(_, flag, __, xmo) \
45 if (netdev->xdp_metadata_ops && netdev->xdp_metadata_ops->xmo) \
46 xdp_rx_meta |= flag;
47 XDP_METADATA_KFUNC_xxx
48 #undef XDP_METADATA_KFUNC
49
50 if (netdev->xsk_tx_metadata_ops) {
51 if (netdev->xsk_tx_metadata_ops->tmo_fill_timestamp)
52 xsk_features |= NETDEV_XSK_FLAGS_TX_TIMESTAMP;
53 if (netdev->xsk_tx_metadata_ops->tmo_request_checksum)
54 xsk_features |= NETDEV_XSK_FLAGS_TX_CHECKSUM;
55 if (netdev->xsk_tx_metadata_ops->tmo_request_launch_time)
56 xsk_features |= NETDEV_XSK_FLAGS_TX_LAUNCH_TIME_FIFO;
57 }
58
59 if (nla_put_u32(rsp, NETDEV_A_DEV_IFINDEX, netdev->ifindex) ||
60 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_FEATURES,
61 netdev->xdp_features, NETDEV_A_DEV_PAD) ||
62 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_RX_METADATA_FEATURES,
63 xdp_rx_meta, NETDEV_A_DEV_PAD) ||
64 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XSK_FEATURES,
65 xsk_features, NETDEV_A_DEV_PAD))
66 goto err_cancel_msg;
67
68 if (netdev->xdp_features & NETDEV_XDP_ACT_XSK_ZEROCOPY) {
69 if (nla_put_u32(rsp, NETDEV_A_DEV_XDP_ZC_MAX_SEGS,
70 netdev->xdp_zc_max_segs))
71 goto err_cancel_msg;
72 }
73
74 genlmsg_end(rsp, hdr);
75
76 return 0;
77
78 err_cancel_msg:
79 genlmsg_cancel(rsp, hdr);
80 return -EMSGSIZE;
81 }
82
83 static void
netdev_genl_dev_notify(struct net_device * netdev,int cmd)84 netdev_genl_dev_notify(struct net_device *netdev, int cmd)
85 {
86 struct genl_info info;
87 struct sk_buff *ntf;
88
89 if (!genl_has_listeners(&netdev_nl_family, dev_net(netdev),
90 NETDEV_NLGRP_MGMT))
91 return;
92
93 genl_info_init_ntf(&info, &netdev_nl_family, cmd);
94
95 ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
96 if (!ntf)
97 return;
98
99 if (netdev_nl_dev_fill(netdev, ntf, &info)) {
100 nlmsg_free(ntf);
101 return;
102 }
103
104 genlmsg_multicast_netns(&netdev_nl_family, dev_net(netdev), ntf,
105 0, NETDEV_NLGRP_MGMT, GFP_KERNEL);
106 }
107
netdev_nl_dev_get_doit(struct sk_buff * skb,struct genl_info * info)108 int netdev_nl_dev_get_doit(struct sk_buff *skb, struct genl_info *info)
109 {
110 struct net_device *netdev;
111 struct sk_buff *rsp;
112 u32 ifindex;
113 int err;
114
115 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX))
116 return -EINVAL;
117
118 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]);
119
120 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
121 if (!rsp)
122 return -ENOMEM;
123
124 rtnl_lock();
125
126 netdev = __dev_get_by_index(genl_info_net(info), ifindex);
127 if (netdev)
128 err = netdev_nl_dev_fill(netdev, rsp, info);
129 else
130 err = -ENODEV;
131
132 rtnl_unlock();
133
134 if (err)
135 goto err_free_msg;
136
137 return genlmsg_reply(rsp, info);
138
139 err_free_msg:
140 nlmsg_free(rsp);
141 return err;
142 }
143
netdev_nl_dev_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)144 int netdev_nl_dev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
145 {
146 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
147 struct net *net = sock_net(skb->sk);
148 struct net_device *netdev;
149 int err = 0;
150
151 rtnl_lock();
152 for_each_netdev_dump(net, netdev, ctx->ifindex) {
153 err = netdev_nl_dev_fill(netdev, skb, genl_info_dump(cb));
154 if (err < 0)
155 break;
156 }
157 rtnl_unlock();
158
159 return err;
160 }
161
162 static int
netdev_nl_napi_fill_one(struct sk_buff * rsp,struct napi_struct * napi,const struct genl_info * info)163 netdev_nl_napi_fill_one(struct sk_buff *rsp, struct napi_struct *napi,
164 const struct genl_info *info)
165 {
166 unsigned long irq_suspend_timeout;
167 unsigned long gro_flush_timeout;
168 u32 napi_defer_hard_irqs;
169 void *hdr;
170 pid_t pid;
171
172 if (!napi->dev->up)
173 return 0;
174
175 hdr = genlmsg_iput(rsp, info);
176 if (!hdr)
177 return -EMSGSIZE;
178
179 if (nla_put_u32(rsp, NETDEV_A_NAPI_ID, napi->napi_id))
180 goto nla_put_failure;
181
182 if (nla_put_u32(rsp, NETDEV_A_NAPI_IFINDEX, napi->dev->ifindex))
183 goto nla_put_failure;
184
185 if (napi->irq >= 0 && nla_put_u32(rsp, NETDEV_A_NAPI_IRQ, napi->irq))
186 goto nla_put_failure;
187
188 if (napi->thread) {
189 pid = task_pid_nr(napi->thread);
190 if (nla_put_u32(rsp, NETDEV_A_NAPI_PID, pid))
191 goto nla_put_failure;
192 }
193
194 napi_defer_hard_irqs = napi_get_defer_hard_irqs(napi);
195 if (nla_put_s32(rsp, NETDEV_A_NAPI_DEFER_HARD_IRQS,
196 napi_defer_hard_irqs))
197 goto nla_put_failure;
198
199 irq_suspend_timeout = napi_get_irq_suspend_timeout(napi);
200 if (nla_put_uint(rsp, NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT,
201 irq_suspend_timeout))
202 goto nla_put_failure;
203
204 gro_flush_timeout = napi_get_gro_flush_timeout(napi);
205 if (nla_put_uint(rsp, NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT,
206 gro_flush_timeout))
207 goto nla_put_failure;
208
209 genlmsg_end(rsp, hdr);
210
211 return 0;
212
213 nla_put_failure:
214 genlmsg_cancel(rsp, hdr);
215 return -EMSGSIZE;
216 }
217
netdev_nl_napi_get_doit(struct sk_buff * skb,struct genl_info * info)218 int netdev_nl_napi_get_doit(struct sk_buff *skb, struct genl_info *info)
219 {
220 struct napi_struct *napi;
221 struct sk_buff *rsp;
222 u32 napi_id;
223 int err;
224
225 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID))
226 return -EINVAL;
227
228 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]);
229
230 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
231 if (!rsp)
232 return -ENOMEM;
233
234 napi = netdev_napi_by_id_lock(genl_info_net(info), napi_id);
235 if (napi) {
236 err = netdev_nl_napi_fill_one(rsp, napi, info);
237 netdev_unlock(napi->dev);
238 } else {
239 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]);
240 err = -ENOENT;
241 }
242
243 if (err) {
244 goto err_free_msg;
245 } else if (!rsp->len) {
246 err = -ENOENT;
247 goto err_free_msg;
248 }
249
250 return genlmsg_reply(rsp, info);
251
252 err_free_msg:
253 nlmsg_free(rsp);
254 return err;
255 }
256
257 static int
netdev_nl_napi_dump_one(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)258 netdev_nl_napi_dump_one(struct net_device *netdev, struct sk_buff *rsp,
259 const struct genl_info *info,
260 struct netdev_nl_dump_ctx *ctx)
261 {
262 struct napi_struct *napi;
263 unsigned int prev_id;
264 int err = 0;
265
266 if (!netdev->up)
267 return err;
268
269 prev_id = UINT_MAX;
270 list_for_each_entry(napi, &netdev->napi_list, dev_list) {
271 if (napi->napi_id < MIN_NAPI_ID)
272 continue;
273
274 /* Dump continuation below depends on the list being sorted */
275 WARN_ON_ONCE(napi->napi_id >= prev_id);
276 prev_id = napi->napi_id;
277
278 if (ctx->napi_id && napi->napi_id >= ctx->napi_id)
279 continue;
280
281 err = netdev_nl_napi_fill_one(rsp, napi, info);
282 if (err)
283 return err;
284 ctx->napi_id = napi->napi_id;
285 }
286 return err;
287 }
288
netdev_nl_napi_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)289 int netdev_nl_napi_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
290 {
291 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
292 const struct genl_info *info = genl_info_dump(cb);
293 struct net *net = sock_net(skb->sk);
294 struct net_device *netdev;
295 u32 ifindex = 0;
296 int err = 0;
297
298 if (info->attrs[NETDEV_A_NAPI_IFINDEX])
299 ifindex = nla_get_u32(info->attrs[NETDEV_A_NAPI_IFINDEX]);
300
301 if (ifindex) {
302 netdev = netdev_get_by_index_lock(net, ifindex);
303 if (netdev) {
304 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx);
305 netdev_unlock(netdev);
306 } else {
307 err = -ENODEV;
308 }
309 } else {
310 for_each_netdev_lock_scoped(net, netdev, ctx->ifindex) {
311 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx);
312 if (err < 0)
313 break;
314 ctx->napi_id = 0;
315 }
316 }
317
318 return err;
319 }
320
321 static int
netdev_nl_napi_set_config(struct napi_struct * napi,struct genl_info * info)322 netdev_nl_napi_set_config(struct napi_struct *napi, struct genl_info *info)
323 {
324 u64 irq_suspend_timeout = 0;
325 u64 gro_flush_timeout = 0;
326 u32 defer = 0;
327
328 if (info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]) {
329 defer = nla_get_u32(info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]);
330 napi_set_defer_hard_irqs(napi, defer);
331 }
332
333 if (info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]) {
334 irq_suspend_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]);
335 napi_set_irq_suspend_timeout(napi, irq_suspend_timeout);
336 }
337
338 if (info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]) {
339 gro_flush_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]);
340 napi_set_gro_flush_timeout(napi, gro_flush_timeout);
341 }
342
343 return 0;
344 }
345
netdev_nl_napi_set_doit(struct sk_buff * skb,struct genl_info * info)346 int netdev_nl_napi_set_doit(struct sk_buff *skb, struct genl_info *info)
347 {
348 struct napi_struct *napi;
349 unsigned int napi_id;
350 int err;
351
352 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID))
353 return -EINVAL;
354
355 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]);
356
357 napi = netdev_napi_by_id_lock(genl_info_net(info), napi_id);
358 if (napi) {
359 err = netdev_nl_napi_set_config(napi, info);
360 netdev_unlock(napi->dev);
361 } else {
362 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]);
363 err = -ENOENT;
364 }
365
366 return err;
367 }
368
369 static int
netdev_nl_queue_fill_one(struct sk_buff * rsp,struct net_device * netdev,u32 q_idx,u32 q_type,const struct genl_info * info)370 netdev_nl_queue_fill_one(struct sk_buff *rsp, struct net_device *netdev,
371 u32 q_idx, u32 q_type, const struct genl_info *info)
372 {
373 struct net_devmem_dmabuf_binding *binding;
374 struct netdev_rx_queue *rxq;
375 struct netdev_queue *txq;
376 void *hdr;
377
378 hdr = genlmsg_iput(rsp, info);
379 if (!hdr)
380 return -EMSGSIZE;
381
382 if (nla_put_u32(rsp, NETDEV_A_QUEUE_ID, q_idx) ||
383 nla_put_u32(rsp, NETDEV_A_QUEUE_TYPE, q_type) ||
384 nla_put_u32(rsp, NETDEV_A_QUEUE_IFINDEX, netdev->ifindex))
385 goto nla_put_failure;
386
387 switch (q_type) {
388 case NETDEV_QUEUE_TYPE_RX:
389 rxq = __netif_get_rx_queue(netdev, q_idx);
390 if (rxq->napi && nla_put_u32(rsp, NETDEV_A_QUEUE_NAPI_ID,
391 rxq->napi->napi_id))
392 goto nla_put_failure;
393
394 binding = rxq->mp_params.mp_priv;
395 if (binding &&
396 nla_put_u32(rsp, NETDEV_A_QUEUE_DMABUF, binding->id))
397 goto nla_put_failure;
398
399 break;
400 case NETDEV_QUEUE_TYPE_TX:
401 txq = netdev_get_tx_queue(netdev, q_idx);
402 if (txq->napi && nla_put_u32(rsp, NETDEV_A_QUEUE_NAPI_ID,
403 txq->napi->napi_id))
404 goto nla_put_failure;
405 }
406
407 genlmsg_end(rsp, hdr);
408
409 return 0;
410
411 nla_put_failure:
412 genlmsg_cancel(rsp, hdr);
413 return -EMSGSIZE;
414 }
415
netdev_nl_queue_validate(struct net_device * netdev,u32 q_id,u32 q_type)416 static int netdev_nl_queue_validate(struct net_device *netdev, u32 q_id,
417 u32 q_type)
418 {
419 switch (q_type) {
420 case NETDEV_QUEUE_TYPE_RX:
421 if (q_id >= netdev->real_num_rx_queues)
422 return -EINVAL;
423 return 0;
424 case NETDEV_QUEUE_TYPE_TX:
425 if (q_id >= netdev->real_num_tx_queues)
426 return -EINVAL;
427 }
428 return 0;
429 }
430
431 static int
netdev_nl_queue_fill(struct sk_buff * rsp,struct net_device * netdev,u32 q_idx,u32 q_type,const struct genl_info * info)432 netdev_nl_queue_fill(struct sk_buff *rsp, struct net_device *netdev, u32 q_idx,
433 u32 q_type, const struct genl_info *info)
434 {
435 int err;
436
437 if (!netdev->up)
438 return -ENOENT;
439
440 err = netdev_nl_queue_validate(netdev, q_idx, q_type);
441 if (err)
442 return err;
443
444 return netdev_nl_queue_fill_one(rsp, netdev, q_idx, q_type, info);
445 }
446
netdev_nl_queue_get_doit(struct sk_buff * skb,struct genl_info * info)447 int netdev_nl_queue_get_doit(struct sk_buff *skb, struct genl_info *info)
448 {
449 u32 q_id, q_type, ifindex;
450 struct net_device *netdev;
451 struct sk_buff *rsp;
452 int err;
453
454 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_ID) ||
455 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_TYPE) ||
456 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_IFINDEX))
457 return -EINVAL;
458
459 q_id = nla_get_u32(info->attrs[NETDEV_A_QUEUE_ID]);
460 q_type = nla_get_u32(info->attrs[NETDEV_A_QUEUE_TYPE]);
461 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]);
462
463 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
464 if (!rsp)
465 return -ENOMEM;
466
467 rtnl_lock();
468
469 netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex);
470 if (netdev) {
471 err = netdev_nl_queue_fill(rsp, netdev, q_id, q_type, info);
472 netdev_unlock(netdev);
473 } else {
474 err = -ENODEV;
475 }
476
477 rtnl_unlock();
478
479 if (err)
480 goto err_free_msg;
481
482 return genlmsg_reply(rsp, info);
483
484 err_free_msg:
485 nlmsg_free(rsp);
486 return err;
487 }
488
489 static int
netdev_nl_queue_dump_one(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)490 netdev_nl_queue_dump_one(struct net_device *netdev, struct sk_buff *rsp,
491 const struct genl_info *info,
492 struct netdev_nl_dump_ctx *ctx)
493 {
494 int err = 0;
495
496 if (!netdev->up)
497 return err;
498
499 for (; ctx->rxq_idx < netdev->real_num_rx_queues; ctx->rxq_idx++) {
500 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->rxq_idx,
501 NETDEV_QUEUE_TYPE_RX, info);
502 if (err)
503 return err;
504 }
505 for (; ctx->txq_idx < netdev->real_num_tx_queues; ctx->txq_idx++) {
506 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->txq_idx,
507 NETDEV_QUEUE_TYPE_TX, info);
508 if (err)
509 return err;
510 }
511
512 return err;
513 }
514
netdev_nl_queue_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)515 int netdev_nl_queue_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
516 {
517 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
518 const struct genl_info *info = genl_info_dump(cb);
519 struct net *net = sock_net(skb->sk);
520 struct net_device *netdev;
521 u32 ifindex = 0;
522 int err = 0;
523
524 if (info->attrs[NETDEV_A_QUEUE_IFINDEX])
525 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]);
526
527 rtnl_lock();
528 if (ifindex) {
529 netdev = netdev_get_by_index_lock(net, ifindex);
530 if (netdev) {
531 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx);
532 netdev_unlock(netdev);
533 } else {
534 err = -ENODEV;
535 }
536 } else {
537 for_each_netdev_lock_scoped(net, netdev, ctx->ifindex) {
538 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx);
539 if (err < 0)
540 break;
541 ctx->rxq_idx = 0;
542 ctx->txq_idx = 0;
543 }
544 }
545 rtnl_unlock();
546
547 return err;
548 }
549
550 #define NETDEV_STAT_NOT_SET (~0ULL)
551
netdev_nl_stats_add(void * _sum,const void * _add,size_t size)552 static void netdev_nl_stats_add(void *_sum, const void *_add, size_t size)
553 {
554 const u64 *add = _add;
555 u64 *sum = _sum;
556
557 while (size) {
558 if (*add != NETDEV_STAT_NOT_SET && *sum != NETDEV_STAT_NOT_SET)
559 *sum += *add;
560 sum++;
561 add++;
562 size -= 8;
563 }
564 }
565
netdev_stat_put(struct sk_buff * rsp,unsigned int attr_id,u64 value)566 static int netdev_stat_put(struct sk_buff *rsp, unsigned int attr_id, u64 value)
567 {
568 if (value == NETDEV_STAT_NOT_SET)
569 return 0;
570 return nla_put_uint(rsp, attr_id, value);
571 }
572
573 static int
netdev_nl_stats_write_rx(struct sk_buff * rsp,struct netdev_queue_stats_rx * rx)574 netdev_nl_stats_write_rx(struct sk_buff *rsp, struct netdev_queue_stats_rx *rx)
575 {
576 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_PACKETS, rx->packets) ||
577 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_BYTES, rx->bytes) ||
578 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_ALLOC_FAIL, rx->alloc_fail) ||
579 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROPS, rx->hw_drops) ||
580 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_OVERRUNS, rx->hw_drop_overruns) ||
581 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_UNNECESSARY, rx->csum_unnecessary) ||
582 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_NONE, rx->csum_none) ||
583 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_BAD, rx->csum_bad) ||
584 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_PACKETS, rx->hw_gro_packets) ||
585 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_BYTES, rx->hw_gro_bytes) ||
586 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_PACKETS, rx->hw_gro_wire_packets) ||
587 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_BYTES, rx->hw_gro_wire_bytes) ||
588 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_RATELIMITS, rx->hw_drop_ratelimits))
589 return -EMSGSIZE;
590 return 0;
591 }
592
593 static int
netdev_nl_stats_write_tx(struct sk_buff * rsp,struct netdev_queue_stats_tx * tx)594 netdev_nl_stats_write_tx(struct sk_buff *rsp, struct netdev_queue_stats_tx *tx)
595 {
596 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_PACKETS, tx->packets) ||
597 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_BYTES, tx->bytes) ||
598 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROPS, tx->hw_drops) ||
599 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_ERRORS, tx->hw_drop_errors) ||
600 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_CSUM_NONE, tx->csum_none) ||
601 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_NEEDS_CSUM, tx->needs_csum) ||
602 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_PACKETS, tx->hw_gso_packets) ||
603 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_BYTES, tx->hw_gso_bytes) ||
604 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_PACKETS, tx->hw_gso_wire_packets) ||
605 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_BYTES, tx->hw_gso_wire_bytes) ||
606 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_RATELIMITS, tx->hw_drop_ratelimits) ||
607 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_STOP, tx->stop) ||
608 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_WAKE, tx->wake))
609 return -EMSGSIZE;
610 return 0;
611 }
612
613 static int
netdev_nl_stats_queue(struct net_device * netdev,struct sk_buff * rsp,u32 q_type,int i,const struct genl_info * info)614 netdev_nl_stats_queue(struct net_device *netdev, struct sk_buff *rsp,
615 u32 q_type, int i, const struct genl_info *info)
616 {
617 const struct netdev_stat_ops *ops = netdev->stat_ops;
618 struct netdev_queue_stats_rx rx;
619 struct netdev_queue_stats_tx tx;
620 void *hdr;
621
622 hdr = genlmsg_iput(rsp, info);
623 if (!hdr)
624 return -EMSGSIZE;
625 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex) ||
626 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_TYPE, q_type) ||
627 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_ID, i))
628 goto nla_put_failure;
629
630 switch (q_type) {
631 case NETDEV_QUEUE_TYPE_RX:
632 memset(&rx, 0xff, sizeof(rx));
633 ops->get_queue_stats_rx(netdev, i, &rx);
634 if (!memchr_inv(&rx, 0xff, sizeof(rx)))
635 goto nla_cancel;
636 if (netdev_nl_stats_write_rx(rsp, &rx))
637 goto nla_put_failure;
638 break;
639 case NETDEV_QUEUE_TYPE_TX:
640 memset(&tx, 0xff, sizeof(tx));
641 ops->get_queue_stats_tx(netdev, i, &tx);
642 if (!memchr_inv(&tx, 0xff, sizeof(tx)))
643 goto nla_cancel;
644 if (netdev_nl_stats_write_tx(rsp, &tx))
645 goto nla_put_failure;
646 break;
647 }
648
649 genlmsg_end(rsp, hdr);
650 return 0;
651
652 nla_cancel:
653 genlmsg_cancel(rsp, hdr);
654 return 0;
655 nla_put_failure:
656 genlmsg_cancel(rsp, hdr);
657 return -EMSGSIZE;
658 }
659
660 static int
netdev_nl_stats_by_queue(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)661 netdev_nl_stats_by_queue(struct net_device *netdev, struct sk_buff *rsp,
662 const struct genl_info *info,
663 struct netdev_nl_dump_ctx *ctx)
664 {
665 const struct netdev_stat_ops *ops = netdev->stat_ops;
666 int i, err;
667
668 if (!(netdev->flags & IFF_UP))
669 return 0;
670
671 i = ctx->rxq_idx;
672 while (ops->get_queue_stats_rx && i < netdev->real_num_rx_queues) {
673 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_RX,
674 i, info);
675 if (err)
676 return err;
677 ctx->rxq_idx = ++i;
678 }
679 i = ctx->txq_idx;
680 while (ops->get_queue_stats_tx && i < netdev->real_num_tx_queues) {
681 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_TX,
682 i, info);
683 if (err)
684 return err;
685 ctx->txq_idx = ++i;
686 }
687
688 ctx->rxq_idx = 0;
689 ctx->txq_idx = 0;
690 return 0;
691 }
692
693 static int
netdev_nl_stats_by_netdev(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info)694 netdev_nl_stats_by_netdev(struct net_device *netdev, struct sk_buff *rsp,
695 const struct genl_info *info)
696 {
697 struct netdev_queue_stats_rx rx_sum, rx;
698 struct netdev_queue_stats_tx tx_sum, tx;
699 const struct netdev_stat_ops *ops;
700 void *hdr;
701 int i;
702
703 ops = netdev->stat_ops;
704 /* Netdev can't guarantee any complete counters */
705 if (!ops->get_base_stats)
706 return 0;
707
708 memset(&rx_sum, 0xff, sizeof(rx_sum));
709 memset(&tx_sum, 0xff, sizeof(tx_sum));
710
711 ops->get_base_stats(netdev, &rx_sum, &tx_sum);
712
713 /* The op was there, but nothing reported, don't bother */
714 if (!memchr_inv(&rx_sum, 0xff, sizeof(rx_sum)) &&
715 !memchr_inv(&tx_sum, 0xff, sizeof(tx_sum)))
716 return 0;
717
718 hdr = genlmsg_iput(rsp, info);
719 if (!hdr)
720 return -EMSGSIZE;
721 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex))
722 goto nla_put_failure;
723
724 for (i = 0; i < netdev->real_num_rx_queues; i++) {
725 memset(&rx, 0xff, sizeof(rx));
726 if (ops->get_queue_stats_rx)
727 ops->get_queue_stats_rx(netdev, i, &rx);
728 netdev_nl_stats_add(&rx_sum, &rx, sizeof(rx));
729 }
730 for (i = 0; i < netdev->real_num_tx_queues; i++) {
731 memset(&tx, 0xff, sizeof(tx));
732 if (ops->get_queue_stats_tx)
733 ops->get_queue_stats_tx(netdev, i, &tx);
734 netdev_nl_stats_add(&tx_sum, &tx, sizeof(tx));
735 }
736
737 if (netdev_nl_stats_write_rx(rsp, &rx_sum) ||
738 netdev_nl_stats_write_tx(rsp, &tx_sum))
739 goto nla_put_failure;
740
741 genlmsg_end(rsp, hdr);
742 return 0;
743
744 nla_put_failure:
745 genlmsg_cancel(rsp, hdr);
746 return -EMSGSIZE;
747 }
748
749 static int
netdev_nl_qstats_get_dump_one(struct net_device * netdev,unsigned int scope,struct sk_buff * skb,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)750 netdev_nl_qstats_get_dump_one(struct net_device *netdev, unsigned int scope,
751 struct sk_buff *skb, const struct genl_info *info,
752 struct netdev_nl_dump_ctx *ctx)
753 {
754 if (!netdev->stat_ops)
755 return 0;
756
757 switch (scope) {
758 case 0:
759 return netdev_nl_stats_by_netdev(netdev, skb, info);
760 case NETDEV_QSTATS_SCOPE_QUEUE:
761 return netdev_nl_stats_by_queue(netdev, skb, info, ctx);
762 }
763
764 return -EINVAL; /* Should not happen, per netlink policy */
765 }
766
netdev_nl_qstats_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)767 int netdev_nl_qstats_get_dumpit(struct sk_buff *skb,
768 struct netlink_callback *cb)
769 {
770 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
771 const struct genl_info *info = genl_info_dump(cb);
772 struct net *net = sock_net(skb->sk);
773 struct net_device *netdev;
774 unsigned int ifindex;
775 unsigned int scope;
776 int err = 0;
777
778 scope = 0;
779 if (info->attrs[NETDEV_A_QSTATS_SCOPE])
780 scope = nla_get_uint(info->attrs[NETDEV_A_QSTATS_SCOPE]);
781
782 ifindex = 0;
783 if (info->attrs[NETDEV_A_QSTATS_IFINDEX])
784 ifindex = nla_get_u32(info->attrs[NETDEV_A_QSTATS_IFINDEX]);
785
786 rtnl_lock();
787 if (ifindex) {
788 netdev = __dev_get_by_index(net, ifindex);
789 if (netdev && netdev->stat_ops) {
790 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb,
791 info, ctx);
792 } else {
793 NL_SET_BAD_ATTR(info->extack,
794 info->attrs[NETDEV_A_QSTATS_IFINDEX]);
795 err = netdev ? -EOPNOTSUPP : -ENODEV;
796 }
797 } else {
798 for_each_netdev_dump(net, netdev, ctx->ifindex) {
799 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb,
800 info, ctx);
801 if (err < 0)
802 break;
803 }
804 }
805 rtnl_unlock();
806
807 return err;
808 }
809
netdev_nl_bind_rx_doit(struct sk_buff * skb,struct genl_info * info)810 int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info)
811 {
812 struct nlattr *tb[ARRAY_SIZE(netdev_queue_id_nl_policy)];
813 struct net_devmem_dmabuf_binding *binding;
814 struct list_head *sock_binding_list;
815 u32 ifindex, dmabuf_fd, rxq_idx;
816 struct net_device *netdev;
817 struct sk_buff *rsp;
818 struct nlattr *attr;
819 int rem, err = 0;
820 void *hdr;
821
822 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX) ||
823 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_FD) ||
824 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_QUEUES))
825 return -EINVAL;
826
827 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]);
828 dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]);
829
830 sock_binding_list = genl_sk_priv_get(&netdev_nl_family,
831 NETLINK_CB(skb).sk);
832 if (IS_ERR(sock_binding_list))
833 return PTR_ERR(sock_binding_list);
834
835 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
836 if (!rsp)
837 return -ENOMEM;
838
839 hdr = genlmsg_iput(rsp, info);
840 if (!hdr) {
841 err = -EMSGSIZE;
842 goto err_genlmsg_free;
843 }
844
845 rtnl_lock();
846
847 netdev = __dev_get_by_index(genl_info_net(info), ifindex);
848 if (!netdev || !netif_device_present(netdev)) {
849 err = -ENODEV;
850 goto err_unlock;
851 }
852
853 if (dev_xdp_prog_count(netdev)) {
854 NL_SET_ERR_MSG(info->extack, "unable to bind dmabuf to device with XDP program attached");
855 err = -EEXIST;
856 goto err_unlock;
857 }
858
859 binding = net_devmem_bind_dmabuf(netdev, dmabuf_fd, info->extack);
860 if (IS_ERR(binding)) {
861 err = PTR_ERR(binding);
862 goto err_unlock;
863 }
864
865 nla_for_each_attr_type(attr, NETDEV_A_DMABUF_QUEUES,
866 genlmsg_data(info->genlhdr),
867 genlmsg_len(info->genlhdr), rem) {
868 err = nla_parse_nested(
869 tb, ARRAY_SIZE(netdev_queue_id_nl_policy) - 1, attr,
870 netdev_queue_id_nl_policy, info->extack);
871 if (err < 0)
872 goto err_unbind;
873
874 if (NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_ID) ||
875 NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_TYPE)) {
876 err = -EINVAL;
877 goto err_unbind;
878 }
879
880 if (nla_get_u32(tb[NETDEV_A_QUEUE_TYPE]) != NETDEV_QUEUE_TYPE_RX) {
881 NL_SET_BAD_ATTR(info->extack, tb[NETDEV_A_QUEUE_TYPE]);
882 err = -EINVAL;
883 goto err_unbind;
884 }
885
886 rxq_idx = nla_get_u32(tb[NETDEV_A_QUEUE_ID]);
887
888 err = net_devmem_bind_dmabuf_to_queue(netdev, rxq_idx, binding,
889 info->extack);
890 if (err)
891 goto err_unbind;
892 }
893
894 list_add(&binding->list, sock_binding_list);
895
896 nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id);
897 genlmsg_end(rsp, hdr);
898
899 err = genlmsg_reply(rsp, info);
900 if (err)
901 goto err_unbind;
902
903 rtnl_unlock();
904
905 return 0;
906
907 err_unbind:
908 net_devmem_unbind_dmabuf(binding);
909 err_unlock:
910 rtnl_unlock();
911 err_genlmsg_free:
912 nlmsg_free(rsp);
913 return err;
914 }
915
netdev_nl_sock_priv_init(struct list_head * priv)916 void netdev_nl_sock_priv_init(struct list_head *priv)
917 {
918 INIT_LIST_HEAD(priv);
919 }
920
netdev_nl_sock_priv_destroy(struct list_head * priv)921 void netdev_nl_sock_priv_destroy(struct list_head *priv)
922 {
923 struct net_devmem_dmabuf_binding *binding;
924 struct net_devmem_dmabuf_binding *temp;
925
926 list_for_each_entry_safe(binding, temp, priv, list) {
927 rtnl_lock();
928 net_devmem_unbind_dmabuf(binding);
929 rtnl_unlock();
930 }
931 }
932
netdev_genl_netdevice_event(struct notifier_block * nb,unsigned long event,void * ptr)933 static int netdev_genl_netdevice_event(struct notifier_block *nb,
934 unsigned long event, void *ptr)
935 {
936 struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
937
938 switch (event) {
939 case NETDEV_REGISTER:
940 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_ADD_NTF);
941 break;
942 case NETDEV_UNREGISTER:
943 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_DEL_NTF);
944 break;
945 case NETDEV_XDP_FEAT_CHANGE:
946 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_CHANGE_NTF);
947 break;
948 }
949
950 return NOTIFY_OK;
951 }
952
953 static struct notifier_block netdev_genl_nb = {
954 .notifier_call = netdev_genl_netdevice_event,
955 };
956
netdev_genl_init(void)957 static int __init netdev_genl_init(void)
958 {
959 int err;
960
961 err = register_netdevice_notifier(&netdev_genl_nb);
962 if (err)
963 return err;
964
965 err = genl_register_family(&netdev_nl_family);
966 if (err)
967 goto err_unreg_ntf;
968
969 return 0;
970
971 err_unreg_ntf:
972 unregister_netdevice_notifier(&netdev_genl_nb);
973 return err;
974 }
975
976 subsys_initcall(netdev_genl_init);
977