1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/errno.h>
4 #include <linux/file.h>
5 #include <linux/slab.h>
6 #include <linux/net.h>
7 #include <linux/compat.h>
8 #include <net/compat.h>
9 #include <linux/io_uring.h>
10
11 #include <uapi/linux/io_uring.h>
12
13 #include "io_uring.h"
14 #include "kbuf.h"
15 #include "alloc_cache.h"
16 #include "net.h"
17 #include "notif.h"
18 #include "rsrc.h"
19
20 #if defined(CONFIG_NET)
21 struct io_shutdown {
22 struct file *file;
23 int how;
24 };
25
26 struct io_accept {
27 struct file *file;
28 struct sockaddr __user *addr;
29 int __user *addr_len;
30 int flags;
31 int iou_flags;
32 u32 file_slot;
33 unsigned long nofile;
34 };
35
36 struct io_socket {
37 struct file *file;
38 int domain;
39 int type;
40 int protocol;
41 int flags;
42 u32 file_slot;
43 unsigned long nofile;
44 };
45
46 struct io_connect {
47 struct file *file;
48 struct sockaddr __user *addr;
49 int addr_len;
50 bool in_progress;
51 bool seen_econnaborted;
52 };
53
54 struct io_bind {
55 struct file *file;
56 int addr_len;
57 };
58
59 struct io_listen {
60 struct file *file;
61 int backlog;
62 };
63
64 struct io_sr_msg {
65 struct file *file;
66 union {
67 struct compat_msghdr __user *umsg_compat;
68 struct user_msghdr __user *umsg;
69 void __user *buf;
70 };
71 int len;
72 unsigned done_io;
73 unsigned msg_flags;
74 unsigned nr_multishot_loops;
75 u16 flags;
76 /* initialised and used only by !msg send variants */
77 u16 buf_group;
78 u16 buf_index;
79 bool retry;
80 bool imported; /* only for io_send_zc */
81 void __user *msg_control;
82 /* used only for send zerocopy */
83 struct io_kiocb *notif;
84 };
85
86 /*
87 * Number of times we'll try and do receives if there's more data. If we
88 * exceed this limit, then add us to the back of the queue and retry from
89 * there. This helps fairness between flooding clients.
90 */
91 #define MULTISHOT_MAX_RETRY 32
92
io_shutdown_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)93 int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
94 {
95 struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown);
96
97 if (unlikely(sqe->off || sqe->addr || sqe->rw_flags ||
98 sqe->buf_index || sqe->splice_fd_in))
99 return -EINVAL;
100
101 shutdown->how = READ_ONCE(sqe->len);
102 req->flags |= REQ_F_FORCE_ASYNC;
103 return 0;
104 }
105
io_shutdown(struct io_kiocb * req,unsigned int issue_flags)106 int io_shutdown(struct io_kiocb *req, unsigned int issue_flags)
107 {
108 struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown);
109 struct socket *sock;
110 int ret;
111
112 WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK);
113
114 sock = sock_from_file(req->file);
115 if (unlikely(!sock))
116 return -ENOTSOCK;
117
118 ret = __sys_shutdown_sock(sock, shutdown->how);
119 io_req_set_res(req, ret, 0);
120 return IOU_OK;
121 }
122
io_net_retry(struct socket * sock,int flags)123 static bool io_net_retry(struct socket *sock, int flags)
124 {
125 if (!(flags & MSG_WAITALL))
126 return false;
127 return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET;
128 }
129
io_netmsg_iovec_free(struct io_async_msghdr * kmsg)130 static void io_netmsg_iovec_free(struct io_async_msghdr *kmsg)
131 {
132 if (kmsg->free_iov) {
133 kfree(kmsg->free_iov);
134 kmsg->free_iov_nr = 0;
135 kmsg->free_iov = NULL;
136 }
137 }
138
io_netmsg_recycle(struct io_kiocb * req,unsigned int issue_flags)139 static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags)
140 {
141 struct io_async_msghdr *hdr = req->async_data;
142
143 /* can't recycle, ensure we free the iovec if we have one */
144 if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) {
145 io_netmsg_iovec_free(hdr);
146 return;
147 }
148
149 /* Let normal cleanup path reap it if we fail adding to the cache */
150 io_alloc_cache_kasan(&hdr->free_iov, &hdr->free_iov_nr);
151 if (io_alloc_cache_put(&req->ctx->netmsg_cache, hdr)) {
152 req->async_data = NULL;
153 req->flags &= ~(REQ_F_ASYNC_DATA|REQ_F_NEED_CLEANUP);
154 }
155 }
156
io_msg_alloc_async(struct io_kiocb * req)157 static struct io_async_msghdr *io_msg_alloc_async(struct io_kiocb *req)
158 {
159 struct io_ring_ctx *ctx = req->ctx;
160 struct io_async_msghdr *hdr;
161
162 hdr = io_uring_alloc_async_data(&ctx->netmsg_cache, req);
163 if (!hdr)
164 return NULL;
165
166 /* If the async data was cached, we might have an iov cached inside. */
167 if (hdr->free_iov)
168 req->flags |= REQ_F_NEED_CLEANUP;
169 return hdr;
170 }
171
172 /* assign new iovec to kmsg, if we need to */
io_net_vec_assign(struct io_kiocb * req,struct io_async_msghdr * kmsg,struct iovec * iov)173 static void io_net_vec_assign(struct io_kiocb *req, struct io_async_msghdr *kmsg,
174 struct iovec *iov)
175 {
176 if (iov) {
177 req->flags |= REQ_F_NEED_CLEANUP;
178 kmsg->free_iov_nr = kmsg->msg.msg_iter.nr_segs;
179 if (kmsg->free_iov)
180 kfree(kmsg->free_iov);
181 kmsg->free_iov = iov;
182 }
183 }
184
io_mshot_prep_retry(struct io_kiocb * req,struct io_async_msghdr * kmsg)185 static inline void io_mshot_prep_retry(struct io_kiocb *req,
186 struct io_async_msghdr *kmsg)
187 {
188 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
189
190 req->flags &= ~REQ_F_BL_EMPTY;
191 sr->done_io = 0;
192 sr->retry = false;
193 sr->len = 0; /* get from the provided buffer */
194 req->buf_index = sr->buf_group;
195 }
196
197 #ifdef CONFIG_COMPAT
io_compat_msg_copy_hdr(struct io_kiocb * req,struct io_async_msghdr * iomsg,struct compat_msghdr * msg,int ddir)198 static int io_compat_msg_copy_hdr(struct io_kiocb *req,
199 struct io_async_msghdr *iomsg,
200 struct compat_msghdr *msg, int ddir)
201 {
202 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
203 struct compat_iovec __user *uiov;
204 struct iovec *iov;
205 int ret, nr_segs;
206
207 if (iomsg->free_iov) {
208 nr_segs = iomsg->free_iov_nr;
209 iov = iomsg->free_iov;
210 } else {
211 iov = &iomsg->fast_iov;
212 nr_segs = 1;
213 }
214
215 if (copy_from_user(msg, sr->umsg_compat, sizeof(*msg)))
216 return -EFAULT;
217
218 uiov = compat_ptr(msg->msg_iov);
219 if (req->flags & REQ_F_BUFFER_SELECT) {
220 compat_ssize_t clen;
221
222 if (msg->msg_iovlen == 0) {
223 sr->len = iov->iov_len = 0;
224 iov->iov_base = NULL;
225 } else if (msg->msg_iovlen > 1) {
226 return -EINVAL;
227 } else {
228 if (!access_ok(uiov, sizeof(*uiov)))
229 return -EFAULT;
230 if (__get_user(clen, &uiov->iov_len))
231 return -EFAULT;
232 if (clen < 0)
233 return -EINVAL;
234 sr->len = clen;
235 }
236
237 return 0;
238 }
239
240 ret = __import_iovec(ddir, (struct iovec __user *)uiov, msg->msg_iovlen,
241 nr_segs, &iov, &iomsg->msg.msg_iter, true);
242 if (unlikely(ret < 0))
243 return ret;
244
245 io_net_vec_assign(req, iomsg, iov);
246 return 0;
247 }
248 #endif
249
io_msg_copy_hdr(struct io_kiocb * req,struct io_async_msghdr * iomsg,struct user_msghdr * msg,int ddir)250 static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,
251 struct user_msghdr *msg, int ddir)
252 {
253 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
254 struct user_msghdr __user *umsg = sr->umsg;
255 struct iovec *iov;
256 int ret, nr_segs;
257
258 if (iomsg->free_iov) {
259 nr_segs = iomsg->free_iov_nr;
260 iov = iomsg->free_iov;
261 } else {
262 iov = &iomsg->fast_iov;
263 nr_segs = 1;
264 }
265
266 if (!user_access_begin(umsg, sizeof(*umsg)))
267 return -EFAULT;
268
269 ret = -EFAULT;
270 unsafe_get_user(msg->msg_name, &umsg->msg_name, ua_end);
271 unsafe_get_user(msg->msg_namelen, &umsg->msg_namelen, ua_end);
272 unsafe_get_user(msg->msg_iov, &umsg->msg_iov, ua_end);
273 unsafe_get_user(msg->msg_iovlen, &umsg->msg_iovlen, ua_end);
274 unsafe_get_user(msg->msg_control, &umsg->msg_control, ua_end);
275 unsafe_get_user(msg->msg_controllen, &umsg->msg_controllen, ua_end);
276 msg->msg_flags = 0;
277
278 if (req->flags & REQ_F_BUFFER_SELECT) {
279 if (msg->msg_iovlen == 0) {
280 sr->len = iov->iov_len = 0;
281 iov->iov_base = NULL;
282 } else if (msg->msg_iovlen > 1) {
283 ret = -EINVAL;
284 goto ua_end;
285 } else {
286 struct iovec __user *uiov = msg->msg_iov;
287
288 /* we only need the length for provided buffers */
289 if (!access_ok(&uiov->iov_len, sizeof(uiov->iov_len)))
290 goto ua_end;
291 unsafe_get_user(iov->iov_len, &uiov->iov_len, ua_end);
292 sr->len = iov->iov_len;
293 }
294 ret = 0;
295 ua_end:
296 user_access_end();
297 return ret;
298 }
299
300 user_access_end();
301 ret = __import_iovec(ddir, msg->msg_iov, msg->msg_iovlen, nr_segs,
302 &iov, &iomsg->msg.msg_iter, false);
303 if (unlikely(ret < 0))
304 return ret;
305
306 io_net_vec_assign(req, iomsg, iov);
307 return 0;
308 }
309
io_sendmsg_copy_hdr(struct io_kiocb * req,struct io_async_msghdr * iomsg)310 static int io_sendmsg_copy_hdr(struct io_kiocb *req,
311 struct io_async_msghdr *iomsg)
312 {
313 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
314 struct user_msghdr msg;
315 int ret;
316
317 iomsg->msg.msg_name = &iomsg->addr;
318 iomsg->msg.msg_iter.nr_segs = 0;
319
320 #ifdef CONFIG_COMPAT
321 if (unlikely(req->ctx->compat)) {
322 struct compat_msghdr cmsg;
323
324 ret = io_compat_msg_copy_hdr(req, iomsg, &cmsg, ITER_SOURCE);
325 if (unlikely(ret))
326 return ret;
327
328 ret = __get_compat_msghdr(&iomsg->msg, &cmsg, NULL);
329 sr->msg_control = iomsg->msg.msg_control_user;
330 return ret;
331 }
332 #endif
333
334 ret = io_msg_copy_hdr(req, iomsg, &msg, ITER_SOURCE);
335 if (unlikely(ret))
336 return ret;
337
338 ret = __copy_msghdr(&iomsg->msg, &msg, NULL);
339
340 /* save msg_control as sys_sendmsg() overwrites it */
341 sr->msg_control = iomsg->msg.msg_control_user;
342 return ret;
343 }
344
io_sendmsg_recvmsg_cleanup(struct io_kiocb * req)345 void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req)
346 {
347 struct io_async_msghdr *io = req->async_data;
348
349 io_netmsg_iovec_free(io);
350 }
351
io_send_setup(struct io_kiocb * req,const struct io_uring_sqe * sqe)352 static int io_send_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe)
353 {
354 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
355 struct io_async_msghdr *kmsg = req->async_data;
356 void __user *addr;
357 u16 addr_len;
358 int ret;
359
360 sr->buf = u64_to_user_ptr(READ_ONCE(sqe->addr));
361
362 if (READ_ONCE(sqe->__pad3[0]))
363 return -EINVAL;
364
365 kmsg->msg.msg_name = NULL;
366 kmsg->msg.msg_namelen = 0;
367 kmsg->msg.msg_control = NULL;
368 kmsg->msg.msg_controllen = 0;
369 kmsg->msg.msg_ubuf = NULL;
370
371 addr = u64_to_user_ptr(READ_ONCE(sqe->addr2));
372 addr_len = READ_ONCE(sqe->addr_len);
373 if (addr) {
374 ret = move_addr_to_kernel(addr, addr_len, &kmsg->addr);
375 if (unlikely(ret < 0))
376 return ret;
377 kmsg->msg.msg_name = &kmsg->addr;
378 kmsg->msg.msg_namelen = addr_len;
379 }
380 if (!io_do_buffer_select(req)) {
381 ret = import_ubuf(ITER_SOURCE, sr->buf, sr->len,
382 &kmsg->msg.msg_iter);
383 if (unlikely(ret < 0))
384 return ret;
385 }
386 return 0;
387 }
388
io_sendmsg_setup(struct io_kiocb * req,const struct io_uring_sqe * sqe)389 static int io_sendmsg_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe)
390 {
391 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
392 struct io_async_msghdr *kmsg = req->async_data;
393 int ret;
394
395 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
396
397 ret = io_sendmsg_copy_hdr(req, kmsg);
398 if (!ret)
399 req->flags |= REQ_F_NEED_CLEANUP;
400 return ret;
401 }
402
403 #define SENDMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_BUNDLE)
404
io_sendmsg_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)405 int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
406 {
407 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
408
409 sr->done_io = 0;
410 sr->retry = false;
411
412 if (req->opcode != IORING_OP_SEND) {
413 if (sqe->addr2 || sqe->file_index)
414 return -EINVAL;
415 }
416
417 sr->len = READ_ONCE(sqe->len);
418 sr->flags = READ_ONCE(sqe->ioprio);
419 if (sr->flags & ~SENDMSG_FLAGS)
420 return -EINVAL;
421 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
422 if (sr->msg_flags & MSG_DONTWAIT)
423 req->flags |= REQ_F_NOWAIT;
424 if (sr->flags & IORING_RECVSEND_BUNDLE) {
425 if (req->opcode == IORING_OP_SENDMSG)
426 return -EINVAL;
427 if (!(req->flags & REQ_F_BUFFER_SELECT))
428 return -EINVAL;
429 sr->msg_flags |= MSG_WAITALL;
430 sr->buf_group = req->buf_index;
431 req->buf_list = NULL;
432 req->flags |= REQ_F_MULTISHOT;
433 }
434
435 #ifdef CONFIG_COMPAT
436 if (req->ctx->compat)
437 sr->msg_flags |= MSG_CMSG_COMPAT;
438 #endif
439 if (unlikely(!io_msg_alloc_async(req)))
440 return -ENOMEM;
441 if (req->opcode != IORING_OP_SENDMSG)
442 return io_send_setup(req, sqe);
443 return io_sendmsg_setup(req, sqe);
444 }
445
io_req_msg_cleanup(struct io_kiocb * req,unsigned int issue_flags)446 static void io_req_msg_cleanup(struct io_kiocb *req,
447 unsigned int issue_flags)
448 {
449 io_netmsg_recycle(req, issue_flags);
450 }
451
452 /*
453 * For bundle completions, we need to figure out how many segments we consumed.
454 * A bundle could be using a single ITER_UBUF if that's all we mapped, or it
455 * could be using an ITER_IOVEC. If the latter, then if we consumed all of
456 * the segments, then it's a trivial questiont o answer. If we have residual
457 * data in the iter, then loop the segments to figure out how much we
458 * transferred.
459 */
io_bundle_nbufs(struct io_async_msghdr * kmsg,int ret)460 static int io_bundle_nbufs(struct io_async_msghdr *kmsg, int ret)
461 {
462 struct iovec *iov;
463 int nbufs;
464
465 /* no data is always zero segments, and a ubuf is always 1 segment */
466 if (ret <= 0)
467 return 0;
468 if (iter_is_ubuf(&kmsg->msg.msg_iter))
469 return 1;
470
471 iov = kmsg->free_iov;
472 if (!iov)
473 iov = &kmsg->fast_iov;
474
475 /* if all data was transferred, it's basic pointer math */
476 if (!iov_iter_count(&kmsg->msg.msg_iter))
477 return iter_iov(&kmsg->msg.msg_iter) - iov;
478
479 /* short transfer, count segments */
480 nbufs = 0;
481 do {
482 int this_len = min_t(int, iov[nbufs].iov_len, ret);
483
484 nbufs++;
485 ret -= this_len;
486 } while (ret);
487
488 return nbufs;
489 }
490
io_send_finish(struct io_kiocb * req,int * ret,struct io_async_msghdr * kmsg,unsigned issue_flags)491 static inline bool io_send_finish(struct io_kiocb *req, int *ret,
492 struct io_async_msghdr *kmsg,
493 unsigned issue_flags)
494 {
495 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
496 bool bundle_finished = *ret <= 0;
497 unsigned int cflags;
498
499 if (!(sr->flags & IORING_RECVSEND_BUNDLE)) {
500 cflags = io_put_kbuf(req, *ret, issue_flags);
501 goto finish;
502 }
503
504 cflags = io_put_kbufs(req, *ret, io_bundle_nbufs(kmsg, *ret), issue_flags);
505
506 if (bundle_finished || req->flags & REQ_F_BL_EMPTY)
507 goto finish;
508
509 /*
510 * Fill CQE for this receive and see if we should keep trying to
511 * receive from this socket.
512 */
513 if (io_req_post_cqe(req, *ret, cflags | IORING_CQE_F_MORE)) {
514 io_mshot_prep_retry(req, kmsg);
515 return false;
516 }
517
518 /* Otherwise stop bundle and use the current result. */
519 finish:
520 io_req_set_res(req, *ret, cflags);
521 *ret = IOU_OK;
522 return true;
523 }
524
io_sendmsg(struct io_kiocb * req,unsigned int issue_flags)525 int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
526 {
527 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
528 struct io_async_msghdr *kmsg = req->async_data;
529 struct socket *sock;
530 unsigned flags;
531 int min_ret = 0;
532 int ret;
533
534 sock = sock_from_file(req->file);
535 if (unlikely(!sock))
536 return -ENOTSOCK;
537
538 if (!(req->flags & REQ_F_POLLED) &&
539 (sr->flags & IORING_RECVSEND_POLL_FIRST))
540 return -EAGAIN;
541
542 flags = sr->msg_flags;
543 if (issue_flags & IO_URING_F_NONBLOCK)
544 flags |= MSG_DONTWAIT;
545 if (flags & MSG_WAITALL)
546 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
547
548 kmsg->msg.msg_control_user = sr->msg_control;
549
550 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
551
552 if (ret < min_ret) {
553 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
554 return -EAGAIN;
555 if (ret > 0 && io_net_retry(sock, flags)) {
556 kmsg->msg.msg_controllen = 0;
557 kmsg->msg.msg_control = NULL;
558 sr->done_io += ret;
559 req->flags |= REQ_F_BL_NO_RECYCLE;
560 return -EAGAIN;
561 }
562 if (ret == -ERESTARTSYS)
563 ret = -EINTR;
564 req_set_fail(req);
565 }
566 io_req_msg_cleanup(req, issue_flags);
567 if (ret >= 0)
568 ret += sr->done_io;
569 else if (sr->done_io)
570 ret = sr->done_io;
571 io_req_set_res(req, ret, 0);
572 return IOU_OK;
573 }
574
io_send_select_buffer(struct io_kiocb * req,unsigned int issue_flags,struct io_async_msghdr * kmsg)575 static int io_send_select_buffer(struct io_kiocb *req, unsigned int issue_flags,
576 struct io_async_msghdr *kmsg)
577 {
578 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
579
580 int ret;
581 struct buf_sel_arg arg = {
582 .iovs = &kmsg->fast_iov,
583 .max_len = min_not_zero(sr->len, INT_MAX),
584 .nr_iovs = 1,
585 };
586
587 if (kmsg->free_iov) {
588 arg.nr_iovs = kmsg->free_iov_nr;
589 arg.iovs = kmsg->free_iov;
590 arg.mode = KBUF_MODE_FREE;
591 }
592
593 if (!(sr->flags & IORING_RECVSEND_BUNDLE))
594 arg.nr_iovs = 1;
595 else
596 arg.mode |= KBUF_MODE_EXPAND;
597
598 ret = io_buffers_select(req, &arg, issue_flags);
599 if (unlikely(ret < 0))
600 return ret;
601
602 if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->free_iov) {
603 kmsg->free_iov_nr = ret;
604 kmsg->free_iov = arg.iovs;
605 req->flags |= REQ_F_NEED_CLEANUP;
606 }
607 sr->len = arg.out_len;
608
609 if (ret == 1) {
610 sr->buf = arg.iovs[0].iov_base;
611 ret = import_ubuf(ITER_SOURCE, sr->buf, sr->len,
612 &kmsg->msg.msg_iter);
613 if (unlikely(ret))
614 return ret;
615 } else {
616 iov_iter_init(&kmsg->msg.msg_iter, ITER_SOURCE,
617 arg.iovs, ret, arg.out_len);
618 }
619
620 return 0;
621 }
622
io_send(struct io_kiocb * req,unsigned int issue_flags)623 int io_send(struct io_kiocb *req, unsigned int issue_flags)
624 {
625 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
626 struct io_async_msghdr *kmsg = req->async_data;
627 struct socket *sock;
628 unsigned flags;
629 int min_ret = 0;
630 int ret;
631
632 sock = sock_from_file(req->file);
633 if (unlikely(!sock))
634 return -ENOTSOCK;
635
636 if (!(req->flags & REQ_F_POLLED) &&
637 (sr->flags & IORING_RECVSEND_POLL_FIRST))
638 return -EAGAIN;
639
640 flags = sr->msg_flags;
641 if (issue_flags & IO_URING_F_NONBLOCK)
642 flags |= MSG_DONTWAIT;
643
644 retry_bundle:
645 if (io_do_buffer_select(req)) {
646 ret = io_send_select_buffer(req, issue_flags, kmsg);
647 if (ret)
648 return ret;
649 }
650
651 /*
652 * If MSG_WAITALL is set, or this is a bundle send, then we need
653 * the full amount. If just bundle is set, if we do a short send
654 * then we complete the bundle sequence rather than continue on.
655 */
656 if (flags & MSG_WAITALL || sr->flags & IORING_RECVSEND_BUNDLE)
657 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
658
659 flags &= ~MSG_INTERNAL_SENDMSG_FLAGS;
660 kmsg->msg.msg_flags = flags;
661 ret = sock_sendmsg(sock, &kmsg->msg);
662 if (ret < min_ret) {
663 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
664 return -EAGAIN;
665
666 if (ret > 0 && io_net_retry(sock, flags)) {
667 sr->len -= ret;
668 sr->buf += ret;
669 sr->done_io += ret;
670 req->flags |= REQ_F_BL_NO_RECYCLE;
671 return -EAGAIN;
672 }
673 if (ret == -ERESTARTSYS)
674 ret = -EINTR;
675 req_set_fail(req);
676 }
677 if (ret >= 0)
678 ret += sr->done_io;
679 else if (sr->done_io)
680 ret = sr->done_io;
681
682 if (!io_send_finish(req, &ret, kmsg, issue_flags))
683 goto retry_bundle;
684
685 io_req_msg_cleanup(req, issue_flags);
686 return ret;
687 }
688
io_recvmsg_mshot_prep(struct io_kiocb * req,struct io_async_msghdr * iomsg,int namelen,size_t controllen)689 static int io_recvmsg_mshot_prep(struct io_kiocb *req,
690 struct io_async_msghdr *iomsg,
691 int namelen, size_t controllen)
692 {
693 if ((req->flags & (REQ_F_APOLL_MULTISHOT|REQ_F_BUFFER_SELECT)) ==
694 (REQ_F_APOLL_MULTISHOT|REQ_F_BUFFER_SELECT)) {
695 int hdr;
696
697 if (unlikely(namelen < 0))
698 return -EOVERFLOW;
699 if (check_add_overflow(sizeof(struct io_uring_recvmsg_out),
700 namelen, &hdr))
701 return -EOVERFLOW;
702 if (check_add_overflow(hdr, controllen, &hdr))
703 return -EOVERFLOW;
704
705 iomsg->namelen = namelen;
706 iomsg->controllen = controllen;
707 return 0;
708 }
709
710 return 0;
711 }
712
io_recvmsg_copy_hdr(struct io_kiocb * req,struct io_async_msghdr * iomsg)713 static int io_recvmsg_copy_hdr(struct io_kiocb *req,
714 struct io_async_msghdr *iomsg)
715 {
716 struct user_msghdr msg;
717 int ret;
718
719 iomsg->msg.msg_name = &iomsg->addr;
720 iomsg->msg.msg_iter.nr_segs = 0;
721
722 #ifdef CONFIG_COMPAT
723 if (unlikely(req->ctx->compat)) {
724 struct compat_msghdr cmsg;
725
726 ret = io_compat_msg_copy_hdr(req, iomsg, &cmsg, ITER_DEST);
727 if (unlikely(ret))
728 return ret;
729
730 ret = __get_compat_msghdr(&iomsg->msg, &cmsg, &iomsg->uaddr);
731 if (unlikely(ret))
732 return ret;
733
734 return io_recvmsg_mshot_prep(req, iomsg, cmsg.msg_namelen,
735 cmsg.msg_controllen);
736 }
737 #endif
738
739 ret = io_msg_copy_hdr(req, iomsg, &msg, ITER_DEST);
740 if (unlikely(ret))
741 return ret;
742
743 ret = __copy_msghdr(&iomsg->msg, &msg, &iomsg->uaddr);
744 if (unlikely(ret))
745 return ret;
746
747 return io_recvmsg_mshot_prep(req, iomsg, msg.msg_namelen,
748 msg.msg_controllen);
749 }
750
io_recvmsg_prep_setup(struct io_kiocb * req)751 static int io_recvmsg_prep_setup(struct io_kiocb *req)
752 {
753 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
754 struct io_async_msghdr *kmsg;
755 int ret;
756
757 kmsg = io_msg_alloc_async(req);
758 if (unlikely(!kmsg))
759 return -ENOMEM;
760
761 if (req->opcode == IORING_OP_RECV) {
762 kmsg->msg.msg_name = NULL;
763 kmsg->msg.msg_namelen = 0;
764 kmsg->msg.msg_inq = 0;
765 kmsg->msg.msg_control = NULL;
766 kmsg->msg.msg_get_inq = 1;
767 kmsg->msg.msg_controllen = 0;
768 kmsg->msg.msg_iocb = NULL;
769 kmsg->msg.msg_ubuf = NULL;
770
771 if (!io_do_buffer_select(req)) {
772 ret = import_ubuf(ITER_DEST, sr->buf, sr->len,
773 &kmsg->msg.msg_iter);
774 if (unlikely(ret))
775 return ret;
776 }
777 return 0;
778 }
779
780 ret = io_recvmsg_copy_hdr(req, kmsg);
781 if (!ret)
782 req->flags |= REQ_F_NEED_CLEANUP;
783 return ret;
784 }
785
786 #define RECVMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT | \
787 IORING_RECVSEND_BUNDLE)
788
io_recvmsg_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)789 int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
790 {
791 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
792
793 sr->done_io = 0;
794 sr->retry = false;
795
796 if (unlikely(sqe->file_index || sqe->addr2))
797 return -EINVAL;
798
799 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
800 sr->len = READ_ONCE(sqe->len);
801 sr->flags = READ_ONCE(sqe->ioprio);
802 if (sr->flags & ~RECVMSG_FLAGS)
803 return -EINVAL;
804 sr->msg_flags = READ_ONCE(sqe->msg_flags);
805 if (sr->msg_flags & MSG_DONTWAIT)
806 req->flags |= REQ_F_NOWAIT;
807 if (sr->msg_flags & MSG_ERRQUEUE)
808 req->flags |= REQ_F_CLEAR_POLLIN;
809 if (req->flags & REQ_F_BUFFER_SELECT) {
810 /*
811 * Store the buffer group for this multishot receive separately,
812 * as if we end up doing an io-wq based issue that selects a
813 * buffer, it has to be committed immediately and that will
814 * clear ->buf_list. This means we lose the link to the buffer
815 * list, and the eventual buffer put on completion then cannot
816 * restore it.
817 */
818 sr->buf_group = req->buf_index;
819 req->buf_list = NULL;
820 }
821 if (sr->flags & IORING_RECV_MULTISHOT) {
822 if (!(req->flags & REQ_F_BUFFER_SELECT))
823 return -EINVAL;
824 if (sr->msg_flags & MSG_WAITALL)
825 return -EINVAL;
826 if (req->opcode == IORING_OP_RECV && sr->len)
827 return -EINVAL;
828 req->flags |= REQ_F_APOLL_MULTISHOT;
829 }
830 if (sr->flags & IORING_RECVSEND_BUNDLE) {
831 if (req->opcode == IORING_OP_RECVMSG)
832 return -EINVAL;
833 }
834
835 #ifdef CONFIG_COMPAT
836 if (req->ctx->compat)
837 sr->msg_flags |= MSG_CMSG_COMPAT;
838 #endif
839 sr->nr_multishot_loops = 0;
840 return io_recvmsg_prep_setup(req);
841 }
842
843 /* bits to clear in old and inherit in new cflags on bundle retry */
844 #define CQE_F_MASK (IORING_CQE_F_SOCK_NONEMPTY|IORING_CQE_F_MORE)
845
846 /*
847 * Finishes io_recv and io_recvmsg.
848 *
849 * Returns true if it is actually finished, or false if it should run
850 * again (for multishot).
851 */
io_recv_finish(struct io_kiocb * req,int * ret,struct io_async_msghdr * kmsg,bool mshot_finished,unsigned issue_flags)852 static inline bool io_recv_finish(struct io_kiocb *req, int *ret,
853 struct io_async_msghdr *kmsg,
854 bool mshot_finished, unsigned issue_flags)
855 {
856 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
857 unsigned int cflags = 0;
858
859 if (kmsg->msg.msg_inq > 0)
860 cflags |= IORING_CQE_F_SOCK_NONEMPTY;
861
862 if (sr->flags & IORING_RECVSEND_BUNDLE) {
863 cflags |= io_put_kbufs(req, *ret, io_bundle_nbufs(kmsg, *ret),
864 issue_flags);
865 if (sr->retry)
866 cflags = req->cqe.flags | (cflags & CQE_F_MASK);
867 /* bundle with no more immediate buffers, we're done */
868 if (req->flags & REQ_F_BL_EMPTY)
869 goto finish;
870 /* if more is available, retry and append to this one */
871 if (!sr->retry && kmsg->msg.msg_inq > 0 && *ret > 0) {
872 req->cqe.flags = cflags & ~CQE_F_MASK;
873 sr->len = kmsg->msg.msg_inq;
874 sr->done_io += *ret;
875 sr->retry = true;
876 return false;
877 }
878 } else {
879 cflags |= io_put_kbuf(req, *ret, issue_flags);
880 }
881
882 /*
883 * Fill CQE for this receive and see if we should keep trying to
884 * receive from this socket.
885 */
886 if ((req->flags & REQ_F_APOLL_MULTISHOT) && !mshot_finished &&
887 io_req_post_cqe(req, *ret, cflags | IORING_CQE_F_MORE)) {
888 int mshot_retry_ret = IOU_ISSUE_SKIP_COMPLETE;
889
890 io_mshot_prep_retry(req, kmsg);
891 /* Known not-empty or unknown state, retry */
892 if (cflags & IORING_CQE_F_SOCK_NONEMPTY || kmsg->msg.msg_inq < 0) {
893 if (sr->nr_multishot_loops++ < MULTISHOT_MAX_RETRY)
894 return false;
895 /* mshot retries exceeded, force a requeue */
896 sr->nr_multishot_loops = 0;
897 mshot_retry_ret = IOU_REQUEUE;
898 }
899 if (issue_flags & IO_URING_F_MULTISHOT)
900 *ret = mshot_retry_ret;
901 else
902 *ret = -EAGAIN;
903 return true;
904 }
905
906 /* Finish the request / stop multishot. */
907 finish:
908 io_req_set_res(req, *ret, cflags);
909
910 if (issue_flags & IO_URING_F_MULTISHOT)
911 *ret = IOU_STOP_MULTISHOT;
912 else
913 *ret = IOU_OK;
914 io_req_msg_cleanup(req, issue_flags);
915 return true;
916 }
917
io_recvmsg_prep_multishot(struct io_async_msghdr * kmsg,struct io_sr_msg * sr,void __user ** buf,size_t * len)918 static int io_recvmsg_prep_multishot(struct io_async_msghdr *kmsg,
919 struct io_sr_msg *sr, void __user **buf,
920 size_t *len)
921 {
922 unsigned long ubuf = (unsigned long) *buf;
923 unsigned long hdr;
924
925 hdr = sizeof(struct io_uring_recvmsg_out) + kmsg->namelen +
926 kmsg->controllen;
927 if (*len < hdr)
928 return -EFAULT;
929
930 if (kmsg->controllen) {
931 unsigned long control = ubuf + hdr - kmsg->controllen;
932
933 kmsg->msg.msg_control_user = (void __user *) control;
934 kmsg->msg.msg_controllen = kmsg->controllen;
935 }
936
937 sr->buf = *buf; /* stash for later copy */
938 *buf = (void __user *) (ubuf + hdr);
939 kmsg->payloadlen = *len = *len - hdr;
940 return 0;
941 }
942
943 struct io_recvmsg_multishot_hdr {
944 struct io_uring_recvmsg_out msg;
945 struct sockaddr_storage addr;
946 };
947
io_recvmsg_multishot(struct socket * sock,struct io_sr_msg * io,struct io_async_msghdr * kmsg,unsigned int flags,bool * finished)948 static int io_recvmsg_multishot(struct socket *sock, struct io_sr_msg *io,
949 struct io_async_msghdr *kmsg,
950 unsigned int flags, bool *finished)
951 {
952 int err;
953 int copy_len;
954 struct io_recvmsg_multishot_hdr hdr;
955
956 if (kmsg->namelen)
957 kmsg->msg.msg_name = &hdr.addr;
958 kmsg->msg.msg_flags = flags & (MSG_CMSG_CLOEXEC|MSG_CMSG_COMPAT);
959 kmsg->msg.msg_namelen = 0;
960
961 if (sock->file->f_flags & O_NONBLOCK)
962 flags |= MSG_DONTWAIT;
963
964 err = sock_recvmsg(sock, &kmsg->msg, flags);
965 *finished = err <= 0;
966 if (err < 0)
967 return err;
968
969 hdr.msg = (struct io_uring_recvmsg_out) {
970 .controllen = kmsg->controllen - kmsg->msg.msg_controllen,
971 .flags = kmsg->msg.msg_flags & ~MSG_CMSG_COMPAT
972 };
973
974 hdr.msg.payloadlen = err;
975 if (err > kmsg->payloadlen)
976 err = kmsg->payloadlen;
977
978 copy_len = sizeof(struct io_uring_recvmsg_out);
979 if (kmsg->msg.msg_namelen > kmsg->namelen)
980 copy_len += kmsg->namelen;
981 else
982 copy_len += kmsg->msg.msg_namelen;
983
984 /*
985 * "fromlen shall refer to the value before truncation.."
986 * 1003.1g
987 */
988 hdr.msg.namelen = kmsg->msg.msg_namelen;
989
990 /* ensure that there is no gap between hdr and sockaddr_storage */
991 BUILD_BUG_ON(offsetof(struct io_recvmsg_multishot_hdr, addr) !=
992 sizeof(struct io_uring_recvmsg_out));
993 if (copy_to_user(io->buf, &hdr, copy_len)) {
994 *finished = true;
995 return -EFAULT;
996 }
997
998 return sizeof(struct io_uring_recvmsg_out) + kmsg->namelen +
999 kmsg->controllen + err;
1000 }
1001
io_recvmsg(struct io_kiocb * req,unsigned int issue_flags)1002 int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
1003 {
1004 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1005 struct io_async_msghdr *kmsg = req->async_data;
1006 struct socket *sock;
1007 unsigned flags;
1008 int ret, min_ret = 0;
1009 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
1010 bool mshot_finished = true;
1011
1012 sock = sock_from_file(req->file);
1013 if (unlikely(!sock))
1014 return -ENOTSOCK;
1015
1016 if (!(req->flags & REQ_F_POLLED) &&
1017 (sr->flags & IORING_RECVSEND_POLL_FIRST))
1018 return -EAGAIN;
1019
1020 flags = sr->msg_flags;
1021 if (force_nonblock)
1022 flags |= MSG_DONTWAIT;
1023
1024 retry_multishot:
1025 if (io_do_buffer_select(req)) {
1026 void __user *buf;
1027 size_t len = sr->len;
1028
1029 buf = io_buffer_select(req, &len, issue_flags);
1030 if (!buf)
1031 return -ENOBUFS;
1032
1033 if (req->flags & REQ_F_APOLL_MULTISHOT) {
1034 ret = io_recvmsg_prep_multishot(kmsg, sr, &buf, &len);
1035 if (ret) {
1036 io_kbuf_recycle(req, issue_flags);
1037 return ret;
1038 }
1039 }
1040
1041 iov_iter_ubuf(&kmsg->msg.msg_iter, ITER_DEST, buf, len);
1042 }
1043
1044 kmsg->msg.msg_get_inq = 1;
1045 kmsg->msg.msg_inq = -1;
1046 if (req->flags & REQ_F_APOLL_MULTISHOT) {
1047 ret = io_recvmsg_multishot(sock, sr, kmsg, flags,
1048 &mshot_finished);
1049 } else {
1050 /* disable partial retry for recvmsg with cmsg attached */
1051 if (flags & MSG_WAITALL && !kmsg->msg.msg_controllen)
1052 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
1053
1054 ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg,
1055 kmsg->uaddr, flags);
1056 }
1057
1058 if (ret < min_ret) {
1059 if (ret == -EAGAIN && force_nonblock) {
1060 if (issue_flags & IO_URING_F_MULTISHOT) {
1061 io_kbuf_recycle(req, issue_flags);
1062 return IOU_ISSUE_SKIP_COMPLETE;
1063 }
1064 return -EAGAIN;
1065 }
1066 if (ret > 0 && io_net_retry(sock, flags)) {
1067 sr->done_io += ret;
1068 req->flags |= REQ_F_BL_NO_RECYCLE;
1069 return -EAGAIN;
1070 }
1071 if (ret == -ERESTARTSYS)
1072 ret = -EINTR;
1073 req_set_fail(req);
1074 } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
1075 req_set_fail(req);
1076 }
1077
1078 if (ret > 0)
1079 ret += sr->done_io;
1080 else if (sr->done_io)
1081 ret = sr->done_io;
1082 else
1083 io_kbuf_recycle(req, issue_flags);
1084
1085 if (!io_recv_finish(req, &ret, kmsg, mshot_finished, issue_flags))
1086 goto retry_multishot;
1087
1088 return ret;
1089 }
1090
io_recv_buf_select(struct io_kiocb * req,struct io_async_msghdr * kmsg,size_t * len,unsigned int issue_flags)1091 static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg,
1092 size_t *len, unsigned int issue_flags)
1093 {
1094 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1095 int ret;
1096
1097 /*
1098 * If the ring isn't locked, then don't use the peek interface
1099 * to grab multiple buffers as we will lock/unlock between
1100 * this selection and posting the buffers.
1101 */
1102 if (!(issue_flags & IO_URING_F_UNLOCKED) &&
1103 sr->flags & IORING_RECVSEND_BUNDLE) {
1104 struct buf_sel_arg arg = {
1105 .iovs = &kmsg->fast_iov,
1106 .nr_iovs = 1,
1107 .mode = KBUF_MODE_EXPAND,
1108 };
1109
1110 if (kmsg->free_iov) {
1111 arg.nr_iovs = kmsg->free_iov_nr;
1112 arg.iovs = kmsg->free_iov;
1113 arg.mode |= KBUF_MODE_FREE;
1114 }
1115
1116 if (kmsg->msg.msg_inq > 0)
1117 arg.max_len = min_not_zero(sr->len, kmsg->msg.msg_inq);
1118
1119 ret = io_buffers_peek(req, &arg);
1120 if (unlikely(ret < 0))
1121 return ret;
1122
1123 /* special case 1 vec, can be a fast path */
1124 if (ret == 1) {
1125 sr->buf = arg.iovs[0].iov_base;
1126 sr->len = arg.iovs[0].iov_len;
1127 goto map_ubuf;
1128 }
1129 iov_iter_init(&kmsg->msg.msg_iter, ITER_DEST, arg.iovs, ret,
1130 arg.out_len);
1131 if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->free_iov) {
1132 kmsg->free_iov_nr = ret;
1133 kmsg->free_iov = arg.iovs;
1134 req->flags |= REQ_F_NEED_CLEANUP;
1135 }
1136 } else {
1137 void __user *buf;
1138
1139 *len = sr->len;
1140 buf = io_buffer_select(req, len, issue_flags);
1141 if (!buf)
1142 return -ENOBUFS;
1143 sr->buf = buf;
1144 sr->len = *len;
1145 map_ubuf:
1146 ret = import_ubuf(ITER_DEST, sr->buf, sr->len,
1147 &kmsg->msg.msg_iter);
1148 if (unlikely(ret))
1149 return ret;
1150 }
1151
1152 return 0;
1153 }
1154
io_recv(struct io_kiocb * req,unsigned int issue_flags)1155 int io_recv(struct io_kiocb *req, unsigned int issue_flags)
1156 {
1157 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1158 struct io_async_msghdr *kmsg = req->async_data;
1159 struct socket *sock;
1160 unsigned flags;
1161 int ret, min_ret = 0;
1162 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
1163 size_t len = sr->len;
1164 bool mshot_finished;
1165
1166 if (!(req->flags & REQ_F_POLLED) &&
1167 (sr->flags & IORING_RECVSEND_POLL_FIRST))
1168 return -EAGAIN;
1169
1170 sock = sock_from_file(req->file);
1171 if (unlikely(!sock))
1172 return -ENOTSOCK;
1173
1174 flags = sr->msg_flags;
1175 if (force_nonblock)
1176 flags |= MSG_DONTWAIT;
1177
1178 retry_multishot:
1179 if (io_do_buffer_select(req)) {
1180 ret = io_recv_buf_select(req, kmsg, &len, issue_flags);
1181 if (unlikely(ret)) {
1182 kmsg->msg.msg_inq = -1;
1183 goto out_free;
1184 }
1185 sr->buf = NULL;
1186 }
1187
1188 kmsg->msg.msg_flags = 0;
1189 kmsg->msg.msg_inq = -1;
1190
1191 if (flags & MSG_WAITALL)
1192 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
1193
1194 ret = sock_recvmsg(sock, &kmsg->msg, flags);
1195 if (ret < min_ret) {
1196 if (ret == -EAGAIN && force_nonblock) {
1197 if (issue_flags & IO_URING_F_MULTISHOT) {
1198 io_kbuf_recycle(req, issue_flags);
1199 return IOU_ISSUE_SKIP_COMPLETE;
1200 }
1201
1202 return -EAGAIN;
1203 }
1204 if (ret > 0 && io_net_retry(sock, flags)) {
1205 sr->len -= ret;
1206 sr->buf += ret;
1207 sr->done_io += ret;
1208 req->flags |= REQ_F_BL_NO_RECYCLE;
1209 return -EAGAIN;
1210 }
1211 if (ret == -ERESTARTSYS)
1212 ret = -EINTR;
1213 req_set_fail(req);
1214 } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
1215 out_free:
1216 req_set_fail(req);
1217 }
1218
1219 mshot_finished = ret <= 0;
1220 if (ret > 0)
1221 ret += sr->done_io;
1222 else if (sr->done_io)
1223 ret = sr->done_io;
1224 else
1225 io_kbuf_recycle(req, issue_flags);
1226
1227 if (!io_recv_finish(req, &ret, kmsg, mshot_finished, issue_flags))
1228 goto retry_multishot;
1229
1230 return ret;
1231 }
1232
io_send_zc_cleanup(struct io_kiocb * req)1233 void io_send_zc_cleanup(struct io_kiocb *req)
1234 {
1235 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);
1236 struct io_async_msghdr *io = req->async_data;
1237
1238 if (req_has_async_data(req))
1239 io_netmsg_iovec_free(io);
1240 if (zc->notif) {
1241 io_notif_flush(zc->notif);
1242 zc->notif = NULL;
1243 }
1244 }
1245
1246 #define IO_ZC_FLAGS_COMMON (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_FIXED_BUF)
1247 #define IO_ZC_FLAGS_VALID (IO_ZC_FLAGS_COMMON | IORING_SEND_ZC_REPORT_USAGE)
1248
io_send_zc_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)1249 int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1250 {
1251 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);
1252 struct io_ring_ctx *ctx = req->ctx;
1253 struct io_kiocb *notif;
1254
1255 zc->done_io = 0;
1256 zc->retry = false;
1257 zc->imported = false;
1258 req->flags |= REQ_F_POLL_NO_LAZY;
1259
1260 if (unlikely(READ_ONCE(sqe->__pad2[0]) || READ_ONCE(sqe->addr3)))
1261 return -EINVAL;
1262 /* we don't support IOSQE_CQE_SKIP_SUCCESS just yet */
1263 if (req->flags & REQ_F_CQE_SKIP)
1264 return -EINVAL;
1265
1266 notif = zc->notif = io_alloc_notif(ctx);
1267 if (!notif)
1268 return -ENOMEM;
1269 notif->cqe.user_data = req->cqe.user_data;
1270 notif->cqe.res = 0;
1271 notif->cqe.flags = IORING_CQE_F_NOTIF;
1272 req->flags |= REQ_F_NEED_CLEANUP;
1273
1274 zc->flags = READ_ONCE(sqe->ioprio);
1275 if (unlikely(zc->flags & ~IO_ZC_FLAGS_COMMON)) {
1276 if (zc->flags & ~IO_ZC_FLAGS_VALID)
1277 return -EINVAL;
1278 if (zc->flags & IORING_SEND_ZC_REPORT_USAGE) {
1279 struct io_notif_data *nd = io_notif_to_data(notif);
1280
1281 nd->zc_report = true;
1282 nd->zc_used = false;
1283 nd->zc_copied = false;
1284 }
1285 }
1286
1287 if (req->opcode != IORING_OP_SEND_ZC) {
1288 if (unlikely(sqe->addr2 || sqe->file_index))
1289 return -EINVAL;
1290 if (unlikely(zc->flags & IORING_RECVSEND_FIXED_BUF))
1291 return -EINVAL;
1292 }
1293
1294 zc->len = READ_ONCE(sqe->len);
1295 zc->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL | MSG_ZEROCOPY;
1296 zc->buf_index = READ_ONCE(sqe->buf_index);
1297 if (zc->msg_flags & MSG_DONTWAIT)
1298 req->flags |= REQ_F_NOWAIT;
1299
1300 #ifdef CONFIG_COMPAT
1301 if (req->ctx->compat)
1302 zc->msg_flags |= MSG_CMSG_COMPAT;
1303 #endif
1304 if (unlikely(!io_msg_alloc_async(req)))
1305 return -ENOMEM;
1306 if (req->opcode != IORING_OP_SENDMSG_ZC)
1307 return io_send_setup(req, sqe);
1308 return io_sendmsg_setup(req, sqe);
1309 }
1310
io_sg_from_iter_iovec(struct sk_buff * skb,struct iov_iter * from,size_t length)1311 static int io_sg_from_iter_iovec(struct sk_buff *skb,
1312 struct iov_iter *from, size_t length)
1313 {
1314 skb_zcopy_downgrade_managed(skb);
1315 return zerocopy_fill_skb_from_iter(skb, from, length);
1316 }
1317
io_sg_from_iter(struct sk_buff * skb,struct iov_iter * from,size_t length)1318 static int io_sg_from_iter(struct sk_buff *skb,
1319 struct iov_iter *from, size_t length)
1320 {
1321 struct skb_shared_info *shinfo = skb_shinfo(skb);
1322 int frag = shinfo->nr_frags;
1323 int ret = 0;
1324 struct bvec_iter bi;
1325 ssize_t copied = 0;
1326 unsigned long truesize = 0;
1327
1328 if (!frag)
1329 shinfo->flags |= SKBFL_MANAGED_FRAG_REFS;
1330 else if (unlikely(!skb_zcopy_managed(skb)))
1331 return zerocopy_fill_skb_from_iter(skb, from, length);
1332
1333 bi.bi_size = min(from->count, length);
1334 bi.bi_bvec_done = from->iov_offset;
1335 bi.bi_idx = 0;
1336
1337 while (bi.bi_size && frag < MAX_SKB_FRAGS) {
1338 struct bio_vec v = mp_bvec_iter_bvec(from->bvec, bi);
1339
1340 copied += v.bv_len;
1341 truesize += PAGE_ALIGN(v.bv_len + v.bv_offset);
1342 __skb_fill_page_desc_noacc(shinfo, frag++, v.bv_page,
1343 v.bv_offset, v.bv_len);
1344 bvec_iter_advance_single(from->bvec, &bi, v.bv_len);
1345 }
1346 if (bi.bi_size)
1347 ret = -EMSGSIZE;
1348
1349 shinfo->nr_frags = frag;
1350 from->bvec += bi.bi_idx;
1351 from->nr_segs -= bi.bi_idx;
1352 from->count -= copied;
1353 from->iov_offset = bi.bi_bvec_done;
1354
1355 skb->data_len += copied;
1356 skb->len += copied;
1357 skb->truesize += truesize;
1358 return ret;
1359 }
1360
io_send_zc_import(struct io_kiocb * req,unsigned int issue_flags)1361 static int io_send_zc_import(struct io_kiocb *req, unsigned int issue_flags)
1362 {
1363 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1364 struct io_async_msghdr *kmsg = req->async_data;
1365 int ret;
1366
1367 if (sr->flags & IORING_RECVSEND_FIXED_BUF) {
1368 struct io_ring_ctx *ctx = req->ctx;
1369 struct io_rsrc_node *node;
1370
1371 ret = -EFAULT;
1372 io_ring_submit_lock(ctx, issue_flags);
1373 node = io_rsrc_node_lookup(&ctx->buf_table, sr->buf_index);
1374 if (node) {
1375 io_req_assign_buf_node(sr->notif, node);
1376 ret = 0;
1377 }
1378 io_ring_submit_unlock(ctx, issue_flags);
1379
1380 if (unlikely(ret))
1381 return ret;
1382
1383 ret = io_import_fixed(ITER_SOURCE, &kmsg->msg.msg_iter,
1384 node->buf, (u64)(uintptr_t)sr->buf,
1385 sr->len);
1386 if (unlikely(ret))
1387 return ret;
1388 kmsg->msg.sg_from_iter = io_sg_from_iter;
1389 } else {
1390 ret = import_ubuf(ITER_SOURCE, sr->buf, sr->len, &kmsg->msg.msg_iter);
1391 if (unlikely(ret))
1392 return ret;
1393 ret = io_notif_account_mem(sr->notif, sr->len);
1394 if (unlikely(ret))
1395 return ret;
1396 kmsg->msg.sg_from_iter = io_sg_from_iter_iovec;
1397 }
1398
1399 return ret;
1400 }
1401
io_send_zc(struct io_kiocb * req,unsigned int issue_flags)1402 int io_send_zc(struct io_kiocb *req, unsigned int issue_flags)
1403 {
1404 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);
1405 struct io_async_msghdr *kmsg = req->async_data;
1406 struct socket *sock;
1407 unsigned msg_flags;
1408 int ret, min_ret = 0;
1409
1410 sock = sock_from_file(req->file);
1411 if (unlikely(!sock))
1412 return -ENOTSOCK;
1413 if (!test_bit(SOCK_SUPPORT_ZC, &sock->flags))
1414 return -EOPNOTSUPP;
1415
1416 if (!(req->flags & REQ_F_POLLED) &&
1417 (zc->flags & IORING_RECVSEND_POLL_FIRST))
1418 return -EAGAIN;
1419
1420 if (!zc->imported) {
1421 zc->imported = true;
1422 ret = io_send_zc_import(req, issue_flags);
1423 if (unlikely(ret))
1424 return ret;
1425 }
1426
1427 msg_flags = zc->msg_flags;
1428 if (issue_flags & IO_URING_F_NONBLOCK)
1429 msg_flags |= MSG_DONTWAIT;
1430 if (msg_flags & MSG_WAITALL)
1431 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
1432 msg_flags &= ~MSG_INTERNAL_SENDMSG_FLAGS;
1433
1434 kmsg->msg.msg_flags = msg_flags;
1435 kmsg->msg.msg_ubuf = &io_notif_to_data(zc->notif)->uarg;
1436 ret = sock_sendmsg(sock, &kmsg->msg);
1437
1438 if (unlikely(ret < min_ret)) {
1439 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
1440 return -EAGAIN;
1441
1442 if (ret > 0 && io_net_retry(sock, kmsg->msg.msg_flags)) {
1443 zc->len -= ret;
1444 zc->buf += ret;
1445 zc->done_io += ret;
1446 req->flags |= REQ_F_BL_NO_RECYCLE;
1447 return -EAGAIN;
1448 }
1449 if (ret == -ERESTARTSYS)
1450 ret = -EINTR;
1451 req_set_fail(req);
1452 }
1453
1454 if (ret >= 0)
1455 ret += zc->done_io;
1456 else if (zc->done_io)
1457 ret = zc->done_io;
1458
1459 /*
1460 * If we're in io-wq we can't rely on tw ordering guarantees, defer
1461 * flushing notif to io_send_zc_cleanup()
1462 */
1463 if (!(issue_flags & IO_URING_F_UNLOCKED)) {
1464 io_notif_flush(zc->notif);
1465 zc->notif = NULL;
1466 io_req_msg_cleanup(req, 0);
1467 }
1468 io_req_set_res(req, ret, IORING_CQE_F_MORE);
1469 return IOU_OK;
1470 }
1471
io_sendmsg_zc(struct io_kiocb * req,unsigned int issue_flags)1472 int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags)
1473 {
1474 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1475 struct io_async_msghdr *kmsg = req->async_data;
1476 struct socket *sock;
1477 unsigned flags;
1478 int ret, min_ret = 0;
1479
1480 sock = sock_from_file(req->file);
1481 if (unlikely(!sock))
1482 return -ENOTSOCK;
1483 if (!test_bit(SOCK_SUPPORT_ZC, &sock->flags))
1484 return -EOPNOTSUPP;
1485
1486 if (!(req->flags & REQ_F_POLLED) &&
1487 (sr->flags & IORING_RECVSEND_POLL_FIRST))
1488 return -EAGAIN;
1489
1490 flags = sr->msg_flags;
1491 if (issue_flags & IO_URING_F_NONBLOCK)
1492 flags |= MSG_DONTWAIT;
1493 if (flags & MSG_WAITALL)
1494 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
1495
1496 kmsg->msg.msg_control_user = sr->msg_control;
1497 kmsg->msg.msg_ubuf = &io_notif_to_data(sr->notif)->uarg;
1498 kmsg->msg.sg_from_iter = io_sg_from_iter_iovec;
1499 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
1500
1501 if (unlikely(ret < min_ret)) {
1502 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
1503 return -EAGAIN;
1504
1505 if (ret > 0 && io_net_retry(sock, flags)) {
1506 sr->done_io += ret;
1507 req->flags |= REQ_F_BL_NO_RECYCLE;
1508 return -EAGAIN;
1509 }
1510 if (ret == -ERESTARTSYS)
1511 ret = -EINTR;
1512 req_set_fail(req);
1513 }
1514
1515 if (ret >= 0)
1516 ret += sr->done_io;
1517 else if (sr->done_io)
1518 ret = sr->done_io;
1519
1520 /*
1521 * If we're in io-wq we can't rely on tw ordering guarantees, defer
1522 * flushing notif to io_send_zc_cleanup()
1523 */
1524 if (!(issue_flags & IO_URING_F_UNLOCKED)) {
1525 io_notif_flush(sr->notif);
1526 sr->notif = NULL;
1527 io_req_msg_cleanup(req, 0);
1528 }
1529 io_req_set_res(req, ret, IORING_CQE_F_MORE);
1530 return IOU_OK;
1531 }
1532
io_sendrecv_fail(struct io_kiocb * req)1533 void io_sendrecv_fail(struct io_kiocb *req)
1534 {
1535 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1536
1537 if (sr->done_io)
1538 req->cqe.res = sr->done_io;
1539
1540 if ((req->flags & REQ_F_NEED_CLEANUP) &&
1541 (req->opcode == IORING_OP_SEND_ZC || req->opcode == IORING_OP_SENDMSG_ZC))
1542 req->cqe.flags |= IORING_CQE_F_MORE;
1543 }
1544
1545 #define ACCEPT_FLAGS (IORING_ACCEPT_MULTISHOT | IORING_ACCEPT_DONTWAIT | \
1546 IORING_ACCEPT_POLL_FIRST)
1547
io_accept_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)1548 int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1549 {
1550 struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept);
1551
1552 if (sqe->len || sqe->buf_index)
1553 return -EINVAL;
1554
1555 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
1556 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
1557 accept->flags = READ_ONCE(sqe->accept_flags);
1558 accept->nofile = rlimit(RLIMIT_NOFILE);
1559 accept->iou_flags = READ_ONCE(sqe->ioprio);
1560 if (accept->iou_flags & ~ACCEPT_FLAGS)
1561 return -EINVAL;
1562
1563 accept->file_slot = READ_ONCE(sqe->file_index);
1564 if (accept->file_slot) {
1565 if (accept->flags & SOCK_CLOEXEC)
1566 return -EINVAL;
1567 if (accept->iou_flags & IORING_ACCEPT_MULTISHOT &&
1568 accept->file_slot != IORING_FILE_INDEX_ALLOC)
1569 return -EINVAL;
1570 }
1571 if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
1572 return -EINVAL;
1573 if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK))
1574 accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK;
1575 if (accept->iou_flags & IORING_ACCEPT_MULTISHOT)
1576 req->flags |= REQ_F_APOLL_MULTISHOT;
1577 if (accept->iou_flags & IORING_ACCEPT_DONTWAIT)
1578 req->flags |= REQ_F_NOWAIT;
1579 return 0;
1580 }
1581
io_accept(struct io_kiocb * req,unsigned int issue_flags)1582 int io_accept(struct io_kiocb *req, unsigned int issue_flags)
1583 {
1584 struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept);
1585 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
1586 bool fixed = !!accept->file_slot;
1587 struct proto_accept_arg arg = {
1588 .flags = force_nonblock ? O_NONBLOCK : 0,
1589 };
1590 struct file *file;
1591 unsigned cflags;
1592 int ret, fd;
1593
1594 if (!(req->flags & REQ_F_POLLED) &&
1595 accept->iou_flags & IORING_ACCEPT_POLL_FIRST)
1596 return -EAGAIN;
1597
1598 retry:
1599 if (!fixed) {
1600 fd = __get_unused_fd_flags(accept->flags, accept->nofile);
1601 if (unlikely(fd < 0))
1602 return fd;
1603 }
1604 arg.err = 0;
1605 arg.is_empty = -1;
1606 file = do_accept(req->file, &arg, accept->addr, accept->addr_len,
1607 accept->flags);
1608 if (IS_ERR(file)) {
1609 if (!fixed)
1610 put_unused_fd(fd);
1611 ret = PTR_ERR(file);
1612 if (ret == -EAGAIN && force_nonblock &&
1613 !(accept->iou_flags & IORING_ACCEPT_DONTWAIT)) {
1614 /*
1615 * if it's multishot and polled, we don't need to
1616 * return EAGAIN to arm the poll infra since it
1617 * has already been done
1618 */
1619 if (issue_flags & IO_URING_F_MULTISHOT)
1620 return IOU_ISSUE_SKIP_COMPLETE;
1621 return ret;
1622 }
1623 if (ret == -ERESTARTSYS)
1624 ret = -EINTR;
1625 req_set_fail(req);
1626 } else if (!fixed) {
1627 fd_install(fd, file);
1628 ret = fd;
1629 } else {
1630 ret = io_fixed_fd_install(req, issue_flags, file,
1631 accept->file_slot);
1632 }
1633
1634 cflags = 0;
1635 if (!arg.is_empty)
1636 cflags |= IORING_CQE_F_SOCK_NONEMPTY;
1637
1638 if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {
1639 io_req_set_res(req, ret, cflags);
1640 return IOU_OK;
1641 }
1642
1643 if (ret < 0)
1644 return ret;
1645 if (io_req_post_cqe(req, ret, cflags | IORING_CQE_F_MORE)) {
1646 if (cflags & IORING_CQE_F_SOCK_NONEMPTY || arg.is_empty == -1)
1647 goto retry;
1648 if (issue_flags & IO_URING_F_MULTISHOT)
1649 return IOU_ISSUE_SKIP_COMPLETE;
1650 return -EAGAIN;
1651 }
1652
1653 io_req_set_res(req, ret, cflags);
1654 if (!(issue_flags & IO_URING_F_MULTISHOT))
1655 return IOU_OK;
1656 return IOU_STOP_MULTISHOT;
1657 }
1658
io_socket_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)1659 int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1660 {
1661 struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket);
1662
1663 if (sqe->addr || sqe->rw_flags || sqe->buf_index)
1664 return -EINVAL;
1665
1666 sock->domain = READ_ONCE(sqe->fd);
1667 sock->type = READ_ONCE(sqe->off);
1668 sock->protocol = READ_ONCE(sqe->len);
1669 sock->file_slot = READ_ONCE(sqe->file_index);
1670 sock->nofile = rlimit(RLIMIT_NOFILE);
1671
1672 sock->flags = sock->type & ~SOCK_TYPE_MASK;
1673 if (sock->file_slot && (sock->flags & SOCK_CLOEXEC))
1674 return -EINVAL;
1675 if (sock->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
1676 return -EINVAL;
1677 return 0;
1678 }
1679
io_socket(struct io_kiocb * req,unsigned int issue_flags)1680 int io_socket(struct io_kiocb *req, unsigned int issue_flags)
1681 {
1682 struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket);
1683 bool fixed = !!sock->file_slot;
1684 struct file *file;
1685 int ret, fd;
1686
1687 if (!fixed) {
1688 fd = __get_unused_fd_flags(sock->flags, sock->nofile);
1689 if (unlikely(fd < 0))
1690 return fd;
1691 }
1692 file = __sys_socket_file(sock->domain, sock->type, sock->protocol);
1693 if (IS_ERR(file)) {
1694 if (!fixed)
1695 put_unused_fd(fd);
1696 ret = PTR_ERR(file);
1697 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
1698 return -EAGAIN;
1699 if (ret == -ERESTARTSYS)
1700 ret = -EINTR;
1701 req_set_fail(req);
1702 } else if (!fixed) {
1703 fd_install(fd, file);
1704 ret = fd;
1705 } else {
1706 ret = io_fixed_fd_install(req, issue_flags, file,
1707 sock->file_slot);
1708 }
1709 io_req_set_res(req, ret, 0);
1710 return IOU_OK;
1711 }
1712
io_connect_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)1713 int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1714 {
1715 struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect);
1716 struct io_async_msghdr *io;
1717
1718 if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in)
1719 return -EINVAL;
1720
1721 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
1722 conn->addr_len = READ_ONCE(sqe->addr2);
1723 conn->in_progress = conn->seen_econnaborted = false;
1724
1725 io = io_msg_alloc_async(req);
1726 if (unlikely(!io))
1727 return -ENOMEM;
1728
1729 return move_addr_to_kernel(conn->addr, conn->addr_len, &io->addr);
1730 }
1731
io_connect(struct io_kiocb * req,unsigned int issue_flags)1732 int io_connect(struct io_kiocb *req, unsigned int issue_flags)
1733 {
1734 struct io_connect *connect = io_kiocb_to_cmd(req, struct io_connect);
1735 struct io_async_msghdr *io = req->async_data;
1736 unsigned file_flags;
1737 int ret;
1738 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
1739
1740 if (unlikely(req->flags & REQ_F_FAIL)) {
1741 ret = -ECONNRESET;
1742 goto out;
1743 }
1744
1745 file_flags = force_nonblock ? O_NONBLOCK : 0;
1746
1747 ret = __sys_connect_file(req->file, &io->addr, connect->addr_len,
1748 file_flags);
1749 if ((ret == -EAGAIN || ret == -EINPROGRESS || ret == -ECONNABORTED)
1750 && force_nonblock) {
1751 if (ret == -EINPROGRESS) {
1752 connect->in_progress = true;
1753 } else if (ret == -ECONNABORTED) {
1754 if (connect->seen_econnaborted)
1755 goto out;
1756 connect->seen_econnaborted = true;
1757 }
1758 return -EAGAIN;
1759 }
1760 if (connect->in_progress) {
1761 /*
1762 * At least bluetooth will return -EBADFD on a re-connect
1763 * attempt, and it's (supposedly) also valid to get -EISCONN
1764 * which means the previous result is good. For both of these,
1765 * grab the sock_error() and use that for the completion.
1766 */
1767 if (ret == -EBADFD || ret == -EISCONN)
1768 ret = sock_error(sock_from_file(req->file)->sk);
1769 }
1770 if (ret == -ERESTARTSYS)
1771 ret = -EINTR;
1772 out:
1773 if (ret < 0)
1774 req_set_fail(req);
1775 io_req_msg_cleanup(req, issue_flags);
1776 io_req_set_res(req, ret, 0);
1777 return IOU_OK;
1778 }
1779
io_bind_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)1780 int io_bind_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1781 {
1782 struct io_bind *bind = io_kiocb_to_cmd(req, struct io_bind);
1783 struct sockaddr __user *uaddr;
1784 struct io_async_msghdr *io;
1785
1786 if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in)
1787 return -EINVAL;
1788
1789 uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr));
1790 bind->addr_len = READ_ONCE(sqe->addr2);
1791
1792 io = io_msg_alloc_async(req);
1793 if (unlikely(!io))
1794 return -ENOMEM;
1795 return move_addr_to_kernel(uaddr, bind->addr_len, &io->addr);
1796 }
1797
io_bind(struct io_kiocb * req,unsigned int issue_flags)1798 int io_bind(struct io_kiocb *req, unsigned int issue_flags)
1799 {
1800 struct io_bind *bind = io_kiocb_to_cmd(req, struct io_bind);
1801 struct io_async_msghdr *io = req->async_data;
1802 struct socket *sock;
1803 int ret;
1804
1805 sock = sock_from_file(req->file);
1806 if (unlikely(!sock))
1807 return -ENOTSOCK;
1808
1809 ret = __sys_bind_socket(sock, &io->addr, bind->addr_len);
1810 if (ret < 0)
1811 req_set_fail(req);
1812 io_req_set_res(req, ret, 0);
1813 return 0;
1814 }
1815
io_listen_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)1816 int io_listen_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1817 {
1818 struct io_listen *listen = io_kiocb_to_cmd(req, struct io_listen);
1819
1820 if (sqe->addr || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in || sqe->addr2)
1821 return -EINVAL;
1822
1823 listen->backlog = READ_ONCE(sqe->len);
1824 return 0;
1825 }
1826
io_listen(struct io_kiocb * req,unsigned int issue_flags)1827 int io_listen(struct io_kiocb *req, unsigned int issue_flags)
1828 {
1829 struct io_listen *listen = io_kiocb_to_cmd(req, struct io_listen);
1830 struct socket *sock;
1831 int ret;
1832
1833 sock = sock_from_file(req->file);
1834 if (unlikely(!sock))
1835 return -ENOTSOCK;
1836
1837 ret = __sys_listen_socket(sock, listen->backlog);
1838 if (ret < 0)
1839 req_set_fail(req);
1840 io_req_set_res(req, ret, 0);
1841 return 0;
1842 }
1843
io_netmsg_cache_free(const void * entry)1844 void io_netmsg_cache_free(const void *entry)
1845 {
1846 struct io_async_msghdr *kmsg = (struct io_async_msghdr *) entry;
1847
1848 if (kmsg->free_iov)
1849 io_netmsg_iovec_free(kmsg);
1850 kfree(kmsg);
1851 }
1852 #endif
1853