Lines Matching +full:num +full:- +full:cs
3 Copyright (C) 2001-2008 Miklos Szeredi <[email protected]>
37 INIT_LIST_HEAD(&req->list); in fuse_request_init()
38 INIT_LIST_HEAD(&req->intr_entry); in fuse_request_init()
39 init_waitqueue_head(&req->waitq); in fuse_request_init()
40 refcount_set(&req->count, 1); in fuse_request_init()
41 __set_bit(FR_PENDING, &req->flags); in fuse_request_init()
42 req->fm = fm; in fuse_request_init()
61 refcount_inc(&req->count); in __fuse_get_request()
67 refcount_dec(&req->count); in __fuse_put_request()
74 fc->initialized = 1; in fuse_set_initialized()
79 return !fc->initialized || (for_background && fc->blocked) || in fuse_block_alloc()
80 (fc->io_uring && fc->connected && !fuse_uring_ready(fc)); in fuse_block_alloc()
86 * lockess check of fc->connected is okay, because atomic_dec_and_test() in fuse_drop_waiting()
88 * to ensure no wake-up is missed. in fuse_drop_waiting()
90 if (atomic_dec_and_test(&fc->num_waiting) && in fuse_drop_waiting()
91 !READ_ONCE(fc->connected)) { in fuse_drop_waiting()
93 wake_up_all(&fc->blocked_waitq); in fuse_drop_waiting()
103 struct fuse_conn *fc = fm->fc; in fuse_get_req()
105 bool no_idmap = !fm->sb || (fm->sb->s_iflags & SB_I_NOIDMAP); in fuse_get_req()
110 atomic_inc(&fc->num_waiting); in fuse_get_req()
113 err = -EINTR; in fuse_get_req()
114 if (wait_event_killable_exclusive(fc->blocked_waitq, in fuse_get_req()
121 err = -ENOTCONN; in fuse_get_req()
122 if (!fc->connected) in fuse_get_req()
125 err = -ECONNREFUSED; in fuse_get_req()
126 if (fc->conn_error) in fuse_get_req()
130 err = -ENOMEM; in fuse_get_req()
133 wake_up(&fc->blocked_waitq); in fuse_get_req()
137 req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns); in fuse_get_req()
139 __set_bit(FR_WAITING, &req->flags); in fuse_get_req()
141 __set_bit(FR_BACKGROUND, &req->flags); in fuse_get_req()
150 * req->in.h.{u,g}id will be equal to FUSE_INVALID_UIDGID. in fuse_get_req()
152 fsuid = no_idmap ? current_fsuid() : mapped_fsuid(idmap, fc->user_ns); in fuse_get_req()
153 fsgid = no_idmap ? current_fsgid() : mapped_fsgid(idmap, fc->user_ns); in fuse_get_req()
154 req->in.h.uid = from_kuid(fc->user_ns, fsuid); in fuse_get_req()
155 req->in.h.gid = from_kgid(fc->user_ns, fsgid); in fuse_get_req()
157 if (no_idmap && unlikely(req->in.h.uid == ((uid_t)-1) || in fuse_get_req()
158 req->in.h.gid == ((gid_t)-1))) { in fuse_get_req()
160 return ERR_PTR(-EOVERFLOW); in fuse_get_req()
172 struct fuse_conn *fc = req->fm->fc; in fuse_put_request()
174 if (refcount_dec_and_test(&req->count)) { in fuse_put_request()
175 if (test_bit(FR_BACKGROUND, &req->flags)) { in fuse_put_request()
180 spin_lock(&fc->bg_lock); in fuse_put_request()
181 if (!fc->blocked) in fuse_put_request()
182 wake_up(&fc->blocked_waitq); in fuse_put_request()
183 spin_unlock(&fc->bg_lock); in fuse_put_request()
186 if (test_bit(FR_WAITING, &req->flags)) { in fuse_put_request()
187 __clear_bit(FR_WAITING, &req->flags); in fuse_put_request()
209 fiq->reqctr += FUSE_REQ_ID_STEP; in fuse_get_unique_locked()
210 return fiq->reqctr; in fuse_get_unique_locked()
217 spin_lock(&fiq->lock); in fuse_get_unique()
219 spin_unlock(&fiq->lock); in fuse_get_unique()
231 * A new request is available, wake fiq->waitq
234 __releases(fiq->lock) in fuse_dev_wake_and_unlock()
236 wake_up(&fiq->waitq); in fuse_dev_wake_and_unlock()
237 kill_fasync(&fiq->fasync, SIGIO, POLL_IN); in fuse_dev_wake_and_unlock()
238 spin_unlock(&fiq->lock); in fuse_dev_wake_and_unlock()
244 spin_lock(&fiq->lock); in fuse_dev_queue_forget()
245 if (fiq->connected) { in fuse_dev_queue_forget()
246 fiq->forget_list_tail->next = forget; in fuse_dev_queue_forget()
247 fiq->forget_list_tail = forget; in fuse_dev_queue_forget()
251 spin_unlock(&fiq->lock); in fuse_dev_queue_forget()
257 spin_lock(&fiq->lock); in fuse_dev_queue_interrupt()
258 if (list_empty(&req->intr_entry)) { in fuse_dev_queue_interrupt()
259 list_add_tail(&req->intr_entry, &fiq->interrupts); in fuse_dev_queue_interrupt()
265 if (test_bit(FR_FINISHED, &req->flags)) { in fuse_dev_queue_interrupt()
266 list_del_init(&req->intr_entry); in fuse_dev_queue_interrupt()
267 spin_unlock(&fiq->lock); in fuse_dev_queue_interrupt()
272 spin_unlock(&fiq->lock); in fuse_dev_queue_interrupt()
278 spin_lock(&fiq->lock); in fuse_dev_queue_req()
279 if (fiq->connected) { in fuse_dev_queue_req()
280 if (req->in.h.opcode != FUSE_NOTIFY_REPLY) in fuse_dev_queue_req()
281 req->in.h.unique = fuse_get_unique_locked(fiq); in fuse_dev_queue_req()
282 list_add_tail(&req->list, &fiq->pending); in fuse_dev_queue_req()
285 spin_unlock(&fiq->lock); in fuse_dev_queue_req()
286 req->out.h.error = -ENOTCONN; in fuse_dev_queue_req()
287 clear_bit(FR_PENDING, &req->flags); in fuse_dev_queue_req()
301 req->in.h.len = sizeof(struct fuse_in_header) + in fuse_send_one()
302 fuse_len_args(req->args->in_numargs, in fuse_send_one()
303 (struct fuse_arg *) req->args->in_args); in fuse_send_one()
305 fiq->ops->send_req(fiq, req); in fuse_send_one()
311 struct fuse_iqueue *fiq = &fc->iq; in fuse_queue_forget()
313 forget->forget_one.nodeid = nodeid; in fuse_queue_forget()
314 forget->forget_one.nlookup = nlookup; in fuse_queue_forget()
316 fiq->ops->send_forget(fiq, forget); in fuse_queue_forget()
321 struct fuse_iqueue *fiq = &fc->iq; in flush_bg_queue()
323 while (fc->active_background < fc->max_background && in flush_bg_queue()
324 !list_empty(&fc->bg_queue)) { in flush_bg_queue()
327 req = list_first_entry(&fc->bg_queue, struct fuse_req, list); in flush_bg_queue()
328 list_del(&req->list); in flush_bg_queue()
329 fc->active_background++; in flush_bg_queue()
344 struct fuse_mount *fm = req->fm; in fuse_request_end()
345 struct fuse_conn *fc = fm->fc; in fuse_request_end()
346 struct fuse_iqueue *fiq = &fc->iq; in fuse_request_end()
348 if (test_and_set_bit(FR_FINISHED, &req->flags)) in fuse_request_end()
357 if (test_bit(FR_INTERRUPTED, &req->flags)) { in fuse_request_end()
358 spin_lock(&fiq->lock); in fuse_request_end()
359 list_del_init(&req->intr_entry); in fuse_request_end()
360 spin_unlock(&fiq->lock); in fuse_request_end()
362 WARN_ON(test_bit(FR_PENDING, &req->flags)); in fuse_request_end()
363 WARN_ON(test_bit(FR_SENT, &req->flags)); in fuse_request_end()
364 if (test_bit(FR_BACKGROUND, &req->flags)) { in fuse_request_end()
365 spin_lock(&fc->bg_lock); in fuse_request_end()
366 clear_bit(FR_BACKGROUND, &req->flags); in fuse_request_end()
367 if (fc->num_background == fc->max_background) { in fuse_request_end()
368 fc->blocked = 0; in fuse_request_end()
369 wake_up(&fc->blocked_waitq); in fuse_request_end()
370 } else if (!fc->blocked) { in fuse_request_end()
374 * fc->blocked with waiters with the wake_up() call in fuse_request_end()
377 if (waitqueue_active(&fc->blocked_waitq)) in fuse_request_end()
378 wake_up(&fc->blocked_waitq); in fuse_request_end()
381 fc->num_background--; in fuse_request_end()
382 fc->active_background--; in fuse_request_end()
384 spin_unlock(&fc->bg_lock); in fuse_request_end()
387 wake_up(&req->waitq); in fuse_request_end()
390 if (test_bit(FR_ASYNC, &req->flags)) in fuse_request_end()
391 req->args->end(fm, req->args, req->out.h.error); in fuse_request_end()
399 struct fuse_iqueue *fiq = &req->fm->fc->iq; in queue_interrupt()
402 if (unlikely(!test_bit(FR_INTERRUPTED, &req->flags))) in queue_interrupt()
403 return -EINVAL; in queue_interrupt()
405 fiq->ops->send_interrupt(fiq, req); in queue_interrupt()
413 if (test_bit(FR_PENDING, &req->flags)) { in fuse_remove_pending_req()
418 list_del(&req->list); in fuse_remove_pending_req()
421 req->out.h.error = -EINTR; in fuse_remove_pending_req()
430 struct fuse_conn *fc = req->fm->fc; in request_wait_answer()
431 struct fuse_iqueue *fiq = &fc->iq; in request_wait_answer()
434 if (!fc->no_interrupt) { in request_wait_answer()
436 err = wait_event_interruptible(req->waitq, in request_wait_answer()
437 test_bit(FR_FINISHED, &req->flags)); in request_wait_answer()
441 set_bit(FR_INTERRUPTED, &req->flags); in request_wait_answer()
444 if (test_bit(FR_SENT, &req->flags)) in request_wait_answer()
448 if (!test_bit(FR_FORCE, &req->flags)) { in request_wait_answer()
452 err = wait_event_killable(req->waitq, in request_wait_answer()
453 test_bit(FR_FINISHED, &req->flags)); in request_wait_answer()
457 if (test_bit(FR_URING, &req->flags)) in request_wait_answer()
460 removed = fuse_remove_pending_req(req, &fiq->lock); in request_wait_answer()
469 wait_event(req->waitq, test_bit(FR_FINISHED, &req->flags)); in request_wait_answer()
474 struct fuse_iqueue *fiq = &req->fm->fc->iq; in __fuse_request_send()
476 BUG_ON(test_bit(FR_BACKGROUND, &req->flags)); in __fuse_request_send()
490 if (fc->minor < 4 && args->opcode == FUSE_STATFS) in fuse_adjust_compat()
491 args->out_args[0].size = FUSE_COMPAT_STATFS_SIZE; in fuse_adjust_compat()
493 if (fc->minor < 9) { in fuse_adjust_compat()
494 switch (args->opcode) { in fuse_adjust_compat()
501 args->out_args[0].size = FUSE_COMPAT_ENTRY_OUT_SIZE; in fuse_adjust_compat()
505 args->out_args[0].size = FUSE_COMPAT_ATTR_OUT_SIZE; in fuse_adjust_compat()
509 if (fc->minor < 12) { in fuse_adjust_compat()
510 switch (args->opcode) { in fuse_adjust_compat()
512 args->in_args[0].size = sizeof(struct fuse_open_in); in fuse_adjust_compat()
515 args->in_args[0].size = FUSE_COMPAT_MKNOD_IN_SIZE; in fuse_adjust_compat()
523 struct fuse_conn *fc = req->fm->fc; in fuse_force_creds()
525 if (!req->fm->sb || req->fm->sb->s_iflags & SB_I_NOIDMAP) { in fuse_force_creds()
526 req->in.h.uid = from_kuid_munged(fc->user_ns, current_fsuid()); in fuse_force_creds()
527 req->in.h.gid = from_kgid_munged(fc->user_ns, current_fsgid()); in fuse_force_creds()
529 req->in.h.uid = FUSE_INVALID_UIDGID; in fuse_force_creds()
530 req->in.h.gid = FUSE_INVALID_UIDGID; in fuse_force_creds()
533 req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns); in fuse_force_creds()
538 req->in.h.opcode = args->opcode; in fuse_args_to_req()
539 req->in.h.nodeid = args->nodeid; in fuse_args_to_req()
540 req->args = args; in fuse_args_to_req()
541 if (args->is_ext) in fuse_args_to_req()
542 req->in.h.total_extlen = args->in_args[args->ext_idx].size / 8; in fuse_args_to_req()
543 if (args->end) in fuse_args_to_req()
544 __set_bit(FR_ASYNC, &req->flags); in fuse_args_to_req()
551 struct fuse_conn *fc = fm->fc; in __fuse_simple_request()
555 if (args->force) { in __fuse_simple_request()
556 atomic_inc(&fc->num_waiting); in __fuse_simple_request()
559 if (!args->nocreds) in __fuse_simple_request()
562 __set_bit(FR_WAITING, &req->flags); in __fuse_simple_request()
563 __set_bit(FR_FORCE, &req->flags); in __fuse_simple_request()
565 WARN_ON(args->nocreds); in __fuse_simple_request()
571 /* Needs to be done after fuse_get_req() so that fc->minor is valid */ in __fuse_simple_request()
575 if (!args->noreply) in __fuse_simple_request()
576 __set_bit(FR_ISREPLY, &req->flags); in __fuse_simple_request()
578 ret = req->out.h.error; in __fuse_simple_request()
579 if (!ret && args->out_argvar) { in __fuse_simple_request()
580 BUG_ON(args->out_numargs == 0); in __fuse_simple_request()
581 ret = args->out_args[args->out_numargs - 1].size; in __fuse_simple_request()
592 struct fuse_iqueue *fiq = &fc->iq; in fuse_request_queue_background_uring()
594 req->in.h.unique = fuse_get_unique(fiq); in fuse_request_queue_background_uring()
595 req->in.h.len = sizeof(struct fuse_in_header) + in fuse_request_queue_background_uring()
596 fuse_len_args(req->args->in_numargs, in fuse_request_queue_background_uring()
597 (struct fuse_arg *) req->args->in_args); in fuse_request_queue_background_uring()
608 struct fuse_mount *fm = req->fm; in fuse_request_queue_background()
609 struct fuse_conn *fc = fm->fc; in fuse_request_queue_background()
612 WARN_ON(!test_bit(FR_BACKGROUND, &req->flags)); in fuse_request_queue_background()
613 if (!test_bit(FR_WAITING, &req->flags)) { in fuse_request_queue_background()
614 __set_bit(FR_WAITING, &req->flags); in fuse_request_queue_background()
615 atomic_inc(&fc->num_waiting); in fuse_request_queue_background()
617 __set_bit(FR_ISREPLY, &req->flags); in fuse_request_queue_background()
624 spin_lock(&fc->bg_lock); in fuse_request_queue_background()
625 if (likely(fc->connected)) { in fuse_request_queue_background()
626 fc->num_background++; in fuse_request_queue_background()
627 if (fc->num_background == fc->max_background) in fuse_request_queue_background()
628 fc->blocked = 1; in fuse_request_queue_background()
629 list_add_tail(&req->list, &fc->bg_queue); in fuse_request_queue_background()
633 spin_unlock(&fc->bg_lock); in fuse_request_queue_background()
643 if (args->force) { in fuse_simple_background()
644 WARN_ON(!args->nocreds); in fuse_simple_background()
647 return -ENOMEM; in fuse_simple_background()
648 __set_bit(FR_BACKGROUND, &req->flags); in fuse_simple_background()
650 WARN_ON(args->nocreds); in fuse_simple_background()
660 return -ENOTCONN; in fuse_simple_background()
671 struct fuse_iqueue *fiq = &fm->fc->iq; in fuse_simple_notify_reply()
677 __clear_bit(FR_ISREPLY, &req->flags); in fuse_simple_notify_reply()
678 req->in.h.unique = unique; in fuse_simple_notify_reply()
689 * anything that could cause a page-fault. If the request was already
696 spin_lock(&req->waitq.lock); in lock_request()
697 if (test_bit(FR_ABORTED, &req->flags)) in lock_request()
698 err = -ENOENT; in lock_request()
700 set_bit(FR_LOCKED, &req->flags); in lock_request()
701 spin_unlock(&req->waitq.lock); in lock_request()
714 spin_lock(&req->waitq.lock); in unlock_request()
715 if (test_bit(FR_ABORTED, &req->flags)) in unlock_request()
716 err = -ENOENT; in unlock_request()
718 clear_bit(FR_LOCKED, &req->flags); in unlock_request()
719 spin_unlock(&req->waitq.lock); in unlock_request()
724 void fuse_copy_init(struct fuse_copy_state *cs, int write, in fuse_copy_init() argument
727 memset(cs, 0, sizeof(*cs)); in fuse_copy_init()
728 cs->write = write; in fuse_copy_init()
729 cs->iter = iter; in fuse_copy_init()
733 static void fuse_copy_finish(struct fuse_copy_state *cs) in fuse_copy_finish() argument
735 if (cs->currbuf) { in fuse_copy_finish()
736 struct pipe_buffer *buf = cs->currbuf; in fuse_copy_finish()
738 if (cs->write) in fuse_copy_finish()
739 buf->len = PAGE_SIZE - cs->len; in fuse_copy_finish()
740 cs->currbuf = NULL; in fuse_copy_finish()
741 } else if (cs->pg) { in fuse_copy_finish()
742 if (cs->write) { in fuse_copy_finish()
743 flush_dcache_page(cs->pg); in fuse_copy_finish()
744 set_page_dirty_lock(cs->pg); in fuse_copy_finish()
746 put_page(cs->pg); in fuse_copy_finish()
748 cs->pg = NULL; in fuse_copy_finish()
755 static int fuse_copy_fill(struct fuse_copy_state *cs) in fuse_copy_fill() argument
760 err = unlock_request(cs->req); in fuse_copy_fill()
764 fuse_copy_finish(cs); in fuse_copy_fill()
765 if (cs->pipebufs) { in fuse_copy_fill()
766 struct pipe_buffer *buf = cs->pipebufs; in fuse_copy_fill()
768 if (!cs->write) { in fuse_copy_fill()
769 err = pipe_buf_confirm(cs->pipe, buf); in fuse_copy_fill()
773 BUG_ON(!cs->nr_segs); in fuse_copy_fill()
774 cs->currbuf = buf; in fuse_copy_fill()
775 cs->pg = buf->page; in fuse_copy_fill()
776 cs->offset = buf->offset; in fuse_copy_fill()
777 cs->len = buf->len; in fuse_copy_fill()
778 cs->pipebufs++; in fuse_copy_fill()
779 cs->nr_segs--; in fuse_copy_fill()
781 if (cs->nr_segs >= cs->pipe->max_usage) in fuse_copy_fill()
782 return -EIO; in fuse_copy_fill()
786 return -ENOMEM; in fuse_copy_fill()
788 buf->page = page; in fuse_copy_fill()
789 buf->offset = 0; in fuse_copy_fill()
790 buf->len = 0; in fuse_copy_fill()
792 cs->currbuf = buf; in fuse_copy_fill()
793 cs->pg = page; in fuse_copy_fill()
794 cs->offset = 0; in fuse_copy_fill()
795 cs->len = PAGE_SIZE; in fuse_copy_fill()
796 cs->pipebufs++; in fuse_copy_fill()
797 cs->nr_segs++; in fuse_copy_fill()
801 err = iov_iter_get_pages2(cs->iter, &page, PAGE_SIZE, 1, &off); in fuse_copy_fill()
805 cs->len = err; in fuse_copy_fill()
806 cs->offset = off; in fuse_copy_fill()
807 cs->pg = page; in fuse_copy_fill()
810 return lock_request(cs->req); in fuse_copy_fill()
814 static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size) in fuse_copy_do() argument
816 unsigned ncpy = min(*size, cs->len); in fuse_copy_do()
818 void *pgaddr = kmap_local_page(cs->pg); in fuse_copy_do()
819 void *buf = pgaddr + cs->offset; in fuse_copy_do()
821 if (cs->write) in fuse_copy_do()
829 *size -= ncpy; in fuse_copy_do()
830 cs->len -= ncpy; in fuse_copy_do()
831 cs->offset += ncpy; in fuse_copy_do()
832 if (cs->is_uring) in fuse_copy_do()
833 cs->ring.copied_sz += ncpy; in fuse_copy_do()
841 folio->mapping != NULL || in fuse_check_folio()
842 (folio->flags & PAGE_FLAGS_CHECK_AT_PREP & in fuse_check_folio()
851 dump_page(&folio->page, "fuse: trying to steal weird page"); in fuse_check_folio()
863 static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep) in fuse_try_move_page() argument
868 struct pipe_buffer *buf = cs->pipebufs; in fuse_try_move_page()
871 err = unlock_request(cs->req); in fuse_try_move_page()
875 fuse_copy_finish(cs); in fuse_try_move_page()
877 err = pipe_buf_confirm(cs->pipe, buf); in fuse_try_move_page()
881 BUG_ON(!cs->nr_segs); in fuse_try_move_page()
882 cs->currbuf = buf; in fuse_try_move_page()
883 cs->len = buf->len; in fuse_try_move_page()
884 cs->pipebufs++; in fuse_try_move_page()
885 cs->nr_segs--; in fuse_try_move_page()
887 if (cs->len != PAGE_SIZE) in fuse_try_move_page()
890 if (!pipe_buf_try_steal(cs->pipe, buf)) in fuse_try_move_page()
893 newfolio = page_folio(buf->page); in fuse_try_move_page()
919 if (!(buf->flags & PIPE_BUF_FLAG_LRU)) in fuse_try_move_page()
926 pipe_buf_release(cs->pipe, buf); in fuse_try_move_page()
929 spin_lock(&cs->req->waitq.lock); in fuse_try_move_page()
930 if (test_bit(FR_ABORTED, &cs->req->flags)) in fuse_try_move_page()
931 err = -ENOENT; in fuse_try_move_page()
933 *pagep = &newfolio->page; in fuse_try_move_page()
934 spin_unlock(&cs->req->waitq.lock); in fuse_try_move_page()
943 /* Drop ref for ap->pages[] array */ in fuse_try_move_page()
945 cs->len = 0; in fuse_try_move_page()
956 cs->pg = buf->page; in fuse_try_move_page()
957 cs->offset = buf->offset; in fuse_try_move_page()
959 err = lock_request(cs->req); in fuse_try_move_page()
966 static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page, in fuse_ref_page() argument
972 if (cs->nr_segs >= cs->pipe->max_usage) in fuse_ref_page()
973 return -EIO; in fuse_ref_page()
976 err = unlock_request(cs->req); in fuse_ref_page()
982 fuse_copy_finish(cs); in fuse_ref_page()
984 buf = cs->pipebufs; in fuse_ref_page()
985 buf->page = page; in fuse_ref_page()
986 buf->offset = offset; in fuse_ref_page()
987 buf->len = count; in fuse_ref_page()
989 cs->pipebufs++; in fuse_ref_page()
990 cs->nr_segs++; in fuse_ref_page()
991 cs->len = 0; in fuse_ref_page()
1000 static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep, in fuse_copy_page() argument
1010 if (cs->write && cs->pipebufs && page) { in fuse_copy_page()
1015 if (cs->req->args->user_pages) { in fuse_copy_page()
1016 err = fuse_copy_fill(cs); in fuse_copy_page()
1020 return fuse_ref_page(cs, page, offset, count); in fuse_copy_page()
1022 } else if (!cs->len) { in fuse_copy_page()
1023 if (cs->move_pages && page && in fuse_copy_page()
1025 err = fuse_try_move_page(cs, pagep); in fuse_copy_page()
1029 err = fuse_copy_fill(cs); in fuse_copy_page()
1037 offset += fuse_copy_do(cs, &buf, &count); in fuse_copy_page()
1040 offset += fuse_copy_do(cs, NULL, &count); in fuse_copy_page()
1042 if (page && !cs->write) in fuse_copy_page()
1048 static int fuse_copy_pages(struct fuse_copy_state *cs, unsigned nbytes, in fuse_copy_pages() argument
1052 struct fuse_req *req = cs->req; in fuse_copy_pages()
1053 struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args); in fuse_copy_pages()
1055 for (i = 0; i < ap->num_folios && (nbytes || zeroing); i++) { in fuse_copy_pages()
1057 unsigned int offset = ap->descs[i].offset; in fuse_copy_pages()
1058 unsigned int count = min(nbytes, ap->descs[i].length); in fuse_copy_pages()
1061 orig = pagep = &ap->folios[i]->page; in fuse_copy_pages()
1063 err = fuse_copy_page(cs, &pagep, offset, count, zeroing); in fuse_copy_pages()
1067 nbytes -= count; in fuse_copy_pages()
1075 ap->folios[i] = page_folio(pagep); in fuse_copy_pages()
1081 static int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size) in fuse_copy_one() argument
1084 if (!cs->len) { in fuse_copy_one()
1085 int err = fuse_copy_fill(cs); in fuse_copy_one()
1089 fuse_copy_do(cs, &val, &size); in fuse_copy_one()
1095 int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs, in fuse_copy_args() argument
1104 if (i == numargs - 1 && argpages) in fuse_copy_args()
1105 err = fuse_copy_pages(cs, arg->size, zeroing); in fuse_copy_args()
1107 err = fuse_copy_one(cs, arg->value, arg->size); in fuse_copy_args()
1114 return fiq->forget_list_head.next != NULL; in forget_pending()
1119 return !list_empty(&fiq->pending) || !list_empty(&fiq->interrupts) || in request_pending()
1129 * Called with fiq->lock held, releases it
1132 struct fuse_copy_state *cs, in fuse_read_interrupt() argument
1134 __releases(fiq->lock) in fuse_read_interrupt()
1141 list_del_init(&req->intr_entry); in fuse_read_interrupt()
1146 ih.unique = (req->in.h.unique | FUSE_INT_REQ_BIT); in fuse_read_interrupt()
1147 arg.unique = req->in.h.unique; in fuse_read_interrupt()
1149 spin_unlock(&fiq->lock); in fuse_read_interrupt()
1151 return -EINVAL; in fuse_read_interrupt()
1153 err = fuse_copy_one(cs, &ih, sizeof(ih)); in fuse_read_interrupt()
1155 err = fuse_copy_one(cs, &arg, sizeof(arg)); in fuse_read_interrupt()
1156 fuse_copy_finish(cs); in fuse_read_interrupt()
1165 struct fuse_forget_link *head = fiq->forget_list_head.next; in fuse_dequeue_forget()
1170 newhead = &(*newhead)->next; in fuse_dequeue_forget()
1172 fiq->forget_list_head.next = *newhead; in fuse_dequeue_forget()
1174 if (fiq->forget_list_head.next == NULL) in fuse_dequeue_forget()
1175 fiq->forget_list_tail = &fiq->forget_list_head; in fuse_dequeue_forget()
1184 struct fuse_copy_state *cs, in fuse_read_single_forget() argument
1186 __releases(fiq->lock) in fuse_read_single_forget()
1191 .nlookup = forget->forget_one.nlookup, in fuse_read_single_forget()
1195 .nodeid = forget->forget_one.nodeid, in fuse_read_single_forget()
1200 spin_unlock(&fiq->lock); in fuse_read_single_forget()
1203 return -EINVAL; in fuse_read_single_forget()
1205 err = fuse_copy_one(cs, &ih, sizeof(ih)); in fuse_read_single_forget()
1207 err = fuse_copy_one(cs, &arg, sizeof(arg)); in fuse_read_single_forget()
1208 fuse_copy_finish(cs); in fuse_read_single_forget()
1217 struct fuse_copy_state *cs, size_t nbytes) in fuse_read_batch_forget() argument
1218 __releases(fiq->lock) in fuse_read_batch_forget()
1232 spin_unlock(&fiq->lock); in fuse_read_batch_forget()
1233 return -EINVAL; in fuse_read_batch_forget()
1236 max_forgets = (nbytes - ih.len) / sizeof(struct fuse_forget_one); in fuse_read_batch_forget()
1238 spin_unlock(&fiq->lock); in fuse_read_batch_forget()
1242 err = fuse_copy_one(cs, &ih, sizeof(ih)); in fuse_read_batch_forget()
1244 err = fuse_copy_one(cs, &arg, sizeof(arg)); in fuse_read_batch_forget()
1250 err = fuse_copy_one(cs, &forget->forget_one, in fuse_read_batch_forget()
1251 sizeof(forget->forget_one)); in fuse_read_batch_forget()
1253 head = forget->next; in fuse_read_batch_forget()
1257 fuse_copy_finish(cs); in fuse_read_batch_forget()
1266 struct fuse_copy_state *cs, in fuse_read_forget() argument
1268 __releases(fiq->lock) in fuse_read_forget()
1270 if (fc->minor < 16 || fiq->forget_list_head.next->next == NULL) in fuse_read_forget()
1271 return fuse_read_single_forget(fiq, cs, nbytes); in fuse_read_forget()
1273 return fuse_read_batch_forget(fiq, cs, nbytes); in fuse_read_forget()
1286 struct fuse_copy_state *cs, size_t nbytes) in fuse_dev_do_read() argument
1289 struct fuse_conn *fc = fud->fc; in fuse_dev_do_read()
1290 struct fuse_iqueue *fiq = &fc->iq; in fuse_dev_do_read()
1291 struct fuse_pqueue *fpq = &fud->pq; in fuse_dev_do_read()
1298 * Require sane minimum read buffer - that has capacity for fixed part in fuse_dev_do_read()
1312 fc->max_write)) in fuse_dev_do_read()
1313 return -EINVAL; in fuse_dev_do_read()
1317 spin_lock(&fiq->lock); in fuse_dev_do_read()
1318 if (!fiq->connected || request_pending(fiq)) in fuse_dev_do_read()
1320 spin_unlock(&fiq->lock); in fuse_dev_do_read()
1322 if (file->f_flags & O_NONBLOCK) in fuse_dev_do_read()
1323 return -EAGAIN; in fuse_dev_do_read()
1324 err = wait_event_interruptible_exclusive(fiq->waitq, in fuse_dev_do_read()
1325 !fiq->connected || request_pending(fiq)); in fuse_dev_do_read()
1330 if (!fiq->connected) { in fuse_dev_do_read()
1331 err = fc->aborted ? -ECONNABORTED : -ENODEV; in fuse_dev_do_read()
1335 if (!list_empty(&fiq->interrupts)) { in fuse_dev_do_read()
1336 req = list_entry(fiq->interrupts.next, struct fuse_req, in fuse_dev_do_read()
1338 return fuse_read_interrupt(fiq, cs, nbytes, req); in fuse_dev_do_read()
1342 if (list_empty(&fiq->pending) || fiq->forget_batch-- > 0) in fuse_dev_do_read()
1343 return fuse_read_forget(fc, fiq, cs, nbytes); in fuse_dev_do_read()
1345 if (fiq->forget_batch <= -8) in fuse_dev_do_read()
1346 fiq->forget_batch = 16; in fuse_dev_do_read()
1349 req = list_entry(fiq->pending.next, struct fuse_req, list); in fuse_dev_do_read()
1350 clear_bit(FR_PENDING, &req->flags); in fuse_dev_do_read()
1351 list_del_init(&req->list); in fuse_dev_do_read()
1352 spin_unlock(&fiq->lock); in fuse_dev_do_read()
1354 args = req->args; in fuse_dev_do_read()
1355 reqsize = req->in.h.len; in fuse_dev_do_read()
1359 req->out.h.error = -EIO; in fuse_dev_do_read()
1361 if (args->opcode == FUSE_SETXATTR) in fuse_dev_do_read()
1362 req->out.h.error = -E2BIG; in fuse_dev_do_read()
1366 spin_lock(&fpq->lock); in fuse_dev_do_read()
1368 * Must not put request on fpq->io queue after having been shut down by in fuse_dev_do_read()
1371 if (!fpq->connected) { in fuse_dev_do_read()
1372 req->out.h.error = err = -ECONNABORTED; in fuse_dev_do_read()
1376 list_add(&req->list, &fpq->io); in fuse_dev_do_read()
1377 spin_unlock(&fpq->lock); in fuse_dev_do_read()
1378 cs->req = req; in fuse_dev_do_read()
1379 err = fuse_copy_one(cs, &req->in.h, sizeof(req->in.h)); in fuse_dev_do_read()
1381 err = fuse_copy_args(cs, args->in_numargs, args->in_pages, in fuse_dev_do_read()
1382 (struct fuse_arg *) args->in_args, 0); in fuse_dev_do_read()
1383 fuse_copy_finish(cs); in fuse_dev_do_read()
1384 spin_lock(&fpq->lock); in fuse_dev_do_read()
1385 clear_bit(FR_LOCKED, &req->flags); in fuse_dev_do_read()
1386 if (!fpq->connected) { in fuse_dev_do_read()
1387 err = fc->aborted ? -ECONNABORTED : -ENODEV; in fuse_dev_do_read()
1391 req->out.h.error = -EIO; in fuse_dev_do_read()
1394 if (!test_bit(FR_ISREPLY, &req->flags)) { in fuse_dev_do_read()
1398 hash = fuse_req_hash(req->in.h.unique); in fuse_dev_do_read()
1399 list_move_tail(&req->list, &fpq->processing[hash]); in fuse_dev_do_read()
1401 set_bit(FR_SENT, &req->flags); in fuse_dev_do_read()
1402 spin_unlock(&fpq->lock); in fuse_dev_do_read()
1405 if (test_bit(FR_INTERRUPTED, &req->flags)) in fuse_dev_do_read()
1412 if (!test_bit(FR_PRIVATE, &req->flags)) in fuse_dev_do_read()
1413 list_del_init(&req->list); in fuse_dev_do_read()
1414 spin_unlock(&fpq->lock); in fuse_dev_do_read()
1419 spin_unlock(&fiq->lock); in fuse_dev_do_read()
1430 file->private_data = NULL; in fuse_dev_open()
1436 struct fuse_copy_state cs; in fuse_dev_read() local
1437 struct file *file = iocb->ki_filp; in fuse_dev_read()
1441 return -EPERM; in fuse_dev_read()
1444 return -EINVAL; in fuse_dev_read()
1446 fuse_copy_init(&cs, 1, to); in fuse_dev_read()
1448 return fuse_dev_do_read(fud, file, &cs, iov_iter_count(to)); in fuse_dev_read()
1458 struct fuse_copy_state cs; in fuse_dev_splice_read() local
1462 return -EPERM; in fuse_dev_splice_read()
1464 bufs = kvmalloc_array(pipe->max_usage, sizeof(struct pipe_buffer), in fuse_dev_splice_read()
1467 return -ENOMEM; in fuse_dev_splice_read()
1469 fuse_copy_init(&cs, 1, NULL); in fuse_dev_splice_read()
1470 cs.pipebufs = bufs; in fuse_dev_splice_read()
1471 cs.pipe = pipe; in fuse_dev_splice_read()
1472 ret = fuse_dev_do_read(fud, in, &cs, len); in fuse_dev_splice_read()
1476 if (pipe_buf_usage(pipe) + cs.nr_segs > pipe->max_usage) { in fuse_dev_splice_read()
1477 ret = -EIO; in fuse_dev_splice_read()
1481 for (ret = total = 0; page_nr < cs.nr_segs; total += ret) { in fuse_dev_splice_read()
1483 * Need to be careful about this. Having buf->ops in module in fuse_dev_splice_read()
1495 for (; page_nr < cs.nr_segs; page_nr++) in fuse_dev_splice_read()
1503 struct fuse_copy_state *cs) in fuse_notify_poll() argument
1506 int err = -EINVAL; in fuse_notify_poll()
1511 err = fuse_copy_one(cs, &outarg, sizeof(outarg)); in fuse_notify_poll()
1515 fuse_copy_finish(cs); in fuse_notify_poll()
1519 fuse_copy_finish(cs); in fuse_notify_poll()
1524 struct fuse_copy_state *cs) in fuse_notify_inval_inode() argument
1527 int err = -EINVAL; in fuse_notify_inval_inode()
1532 err = fuse_copy_one(cs, &outarg, sizeof(outarg)); in fuse_notify_inval_inode()
1535 fuse_copy_finish(cs); in fuse_notify_inval_inode()
1537 down_read(&fc->killsb); in fuse_notify_inval_inode()
1540 up_read(&fc->killsb); in fuse_notify_inval_inode()
1544 fuse_copy_finish(cs); in fuse_notify_inval_inode()
1549 struct fuse_copy_state *cs) in fuse_notify_inval_entry() argument
1552 int err = -ENOMEM; in fuse_notify_inval_entry()
1560 err = -EINVAL; in fuse_notify_inval_entry()
1564 err = fuse_copy_one(cs, &outarg, sizeof(outarg)); in fuse_notify_inval_entry()
1568 err = -ENAMETOOLONG; in fuse_notify_inval_entry()
1572 err = -EINVAL; in fuse_notify_inval_entry()
1578 err = fuse_copy_one(cs, buf, outarg.namelen + 1); in fuse_notify_inval_entry()
1581 fuse_copy_finish(cs); in fuse_notify_inval_entry()
1584 down_read(&fc->killsb); in fuse_notify_inval_entry()
1586 up_read(&fc->killsb); in fuse_notify_inval_entry()
1592 fuse_copy_finish(cs); in fuse_notify_inval_entry()
1597 struct fuse_copy_state *cs) in fuse_notify_delete() argument
1600 int err = -ENOMEM; in fuse_notify_delete()
1608 err = -EINVAL; in fuse_notify_delete()
1612 err = fuse_copy_one(cs, &outarg, sizeof(outarg)); in fuse_notify_delete()
1616 err = -ENAMETOOLONG; in fuse_notify_delete()
1620 err = -EINVAL; in fuse_notify_delete()
1626 err = fuse_copy_one(cs, buf, outarg.namelen + 1); in fuse_notify_delete()
1629 fuse_copy_finish(cs); in fuse_notify_delete()
1632 down_read(&fc->killsb); in fuse_notify_delete()
1634 up_read(&fc->killsb); in fuse_notify_delete()
1640 fuse_copy_finish(cs); in fuse_notify_delete()
1645 struct fuse_copy_state *cs) in fuse_notify_store() argument
1654 unsigned int num; in fuse_notify_store() local
1658 err = -EINVAL; in fuse_notify_store()
1662 err = fuse_copy_one(cs, &outarg, sizeof(outarg)); in fuse_notify_store()
1666 err = -EINVAL; in fuse_notify_store()
1667 if (size - sizeof(outarg) != outarg.size) in fuse_notify_store()
1672 down_read(&fc->killsb); in fuse_notify_store()
1674 err = -ENOENT; in fuse_notify_store()
1679 mapping = inode->i_mapping; in fuse_notify_store()
1689 num = outarg.size; in fuse_notify_store()
1690 while (num) { in fuse_notify_store()
1700 page = &folio->page; in fuse_notify_store()
1701 this_num = min_t(unsigned, num, folio_size(folio) - offset); in fuse_notify_store()
1702 err = fuse_copy_page(cs, &page, offset, this_num, 0); in fuse_notify_store()
1714 num -= this_num; in fuse_notify_store()
1724 up_read(&fc->killsb); in fuse_notify_store()
1726 fuse_copy_finish(cs); in fuse_notify_store()
1741 release_pages(ra->ap.folios, ra->ap.num_folios); in fuse_retrieve_end()
1749 struct address_space *mapping = inode->i_mapping; in fuse_retrieve()
1752 unsigned int num; in fuse_retrieve() local
1756 struct fuse_conn *fc = fm->fc; in fuse_retrieve()
1762 offset = outarg->offset & ~PAGE_MASK; in fuse_retrieve()
1765 num = min(outarg->size, fc->max_write); in fuse_retrieve()
1766 if (outarg->offset > file_size) in fuse_retrieve()
1767 num = 0; in fuse_retrieve()
1768 else if (outarg->offset + num > file_size) in fuse_retrieve()
1769 num = file_size - outarg->offset; in fuse_retrieve()
1771 num_pages = (num + offset + PAGE_SIZE - 1) >> PAGE_SHIFT; in fuse_retrieve()
1772 num_pages = min(num_pages, fc->max_pages); in fuse_retrieve()
1774 args_size += num_pages * (sizeof(ap->folios[0]) + sizeof(ap->descs[0])); in fuse_retrieve()
1778 return -ENOMEM; in fuse_retrieve()
1780 ap = &ra->ap; in fuse_retrieve()
1781 ap->folios = (void *) (ra + 1); in fuse_retrieve()
1782 ap->descs = (void *) (ap->folios + num_pages); in fuse_retrieve()
1784 args = &ap->args; in fuse_retrieve()
1785 args->nodeid = outarg->nodeid; in fuse_retrieve()
1786 args->opcode = FUSE_NOTIFY_REPLY; in fuse_retrieve()
1787 args->in_numargs = 3; in fuse_retrieve()
1788 args->in_pages = true; in fuse_retrieve()
1789 args->end = fuse_retrieve_end; in fuse_retrieve()
1791 index = outarg->offset >> PAGE_SHIFT; in fuse_retrieve()
1793 while (num && cur_pages < num_pages) { in fuse_retrieve()
1801 this_num = min_t(unsigned, num, PAGE_SIZE - offset); in fuse_retrieve()
1802 ap->folios[ap->num_folios] = folio; in fuse_retrieve()
1803 ap->descs[ap->num_folios].offset = offset; in fuse_retrieve()
1804 ap->descs[ap->num_folios].length = this_num; in fuse_retrieve()
1805 ap->num_folios++; in fuse_retrieve()
1809 num -= this_num; in fuse_retrieve()
1813 ra->inarg.offset = outarg->offset; in fuse_retrieve()
1814 ra->inarg.size = total_len; in fuse_retrieve()
1816 args->in_args[1].size = sizeof(ra->inarg); in fuse_retrieve()
1817 args->in_args[1].value = &ra->inarg; in fuse_retrieve()
1818 args->in_args[2].size = total_len; in fuse_retrieve()
1820 err = fuse_simple_notify_reply(fm, args, outarg->notify_unique); in fuse_retrieve()
1828 struct fuse_copy_state *cs) in fuse_notify_retrieve() argument
1836 err = -EINVAL; in fuse_notify_retrieve()
1840 err = fuse_copy_one(cs, &outarg, sizeof(outarg)); in fuse_notify_retrieve()
1844 fuse_copy_finish(cs); in fuse_notify_retrieve()
1846 down_read(&fc->killsb); in fuse_notify_retrieve()
1847 err = -ENOENT; in fuse_notify_retrieve()
1855 up_read(&fc->killsb); in fuse_notify_retrieve()
1860 fuse_copy_finish(cs); in fuse_notify_retrieve()
1875 * non-idempotent requests.
1881 struct fuse_iqueue *fiq = &fc->iq; in fuse_resend()
1885 spin_lock(&fc->lock); in fuse_resend()
1886 if (!fc->connected) { in fuse_resend()
1887 spin_unlock(&fc->lock); in fuse_resend()
1891 list_for_each_entry(fud, &fc->devices, entry) { in fuse_resend()
1892 struct fuse_pqueue *fpq = &fud->pq; in fuse_resend()
1894 spin_lock(&fpq->lock); in fuse_resend()
1896 list_splice_tail_init(&fpq->processing[i], &to_queue); in fuse_resend()
1897 spin_unlock(&fpq->lock); in fuse_resend()
1899 spin_unlock(&fc->lock); in fuse_resend()
1902 set_bit(FR_PENDING, &req->flags); in fuse_resend()
1903 clear_bit(FR_SENT, &req->flags); in fuse_resend()
1905 req->in.h.unique |= FUSE_UNIQUE_RESEND; in fuse_resend()
1908 spin_lock(&fiq->lock); in fuse_resend()
1909 if (!fiq->connected) { in fuse_resend()
1910 spin_unlock(&fiq->lock); in fuse_resend()
1912 clear_bit(FR_PENDING, &req->flags); in fuse_resend()
1917 list_splice(&to_queue, &fiq->pending); in fuse_resend()
1928 unsigned int size, struct fuse_copy_state *cs) in fuse_notify() argument
1931 cs->move_pages = 0; in fuse_notify()
1935 return fuse_notify_poll(fc, size, cs); in fuse_notify()
1938 return fuse_notify_inval_inode(fc, size, cs); in fuse_notify()
1941 return fuse_notify_inval_entry(fc, size, cs); in fuse_notify()
1944 return fuse_notify_store(fc, size, cs); in fuse_notify()
1947 return fuse_notify_retrieve(fc, size, cs); in fuse_notify()
1950 return fuse_notify_delete(fc, size, cs); in fuse_notify()
1956 fuse_copy_finish(cs); in fuse_notify()
1957 return -EINVAL; in fuse_notify()
1967 list_for_each_entry(req, &fpq->processing[hash], list) { in fuse_request_find()
1968 if (req->in.h.unique == unique) in fuse_request_find()
1974 int fuse_copy_out_args(struct fuse_copy_state *cs, struct fuse_args *args, in fuse_copy_out_args() argument
1981 * Uring has all headers separated from args - args is payload only in fuse_copy_out_args()
1983 if (!cs->is_uring) in fuse_copy_out_args()
1986 reqsize += fuse_len_args(args->out_numargs, args->out_args); in fuse_copy_out_args()
1988 if (reqsize < nbytes || (reqsize > nbytes && !args->out_argvar)) in fuse_copy_out_args()
1989 return -EINVAL; in fuse_copy_out_args()
1991 struct fuse_arg *lastarg = &args->out_args[args->out_numargs-1]; in fuse_copy_out_args()
1992 unsigned diffsize = reqsize - nbytes; in fuse_copy_out_args()
1994 if (diffsize > lastarg->size) in fuse_copy_out_args()
1995 return -EINVAL; in fuse_copy_out_args()
1996 lastarg->size -= diffsize; in fuse_copy_out_args()
1998 return fuse_copy_args(cs, args->out_numargs, args->out_pages, in fuse_copy_out_args()
1999 args->out_args, args->page_zeroing); in fuse_copy_out_args()
2010 struct fuse_copy_state *cs, size_t nbytes) in fuse_dev_do_write() argument
2013 struct fuse_conn *fc = fud->fc; in fuse_dev_do_write()
2014 struct fuse_pqueue *fpq = &fud->pq; in fuse_dev_do_write()
2018 err = -EINVAL; in fuse_dev_do_write()
2022 err = fuse_copy_one(cs, &oh, sizeof(oh)); in fuse_dev_do_write()
2026 err = -EINVAL; in fuse_dev_do_write()
2035 err = fuse_notify(fc, oh.error, nbytes - sizeof(oh), cs); in fuse_dev_do_write()
2039 err = -EINVAL; in fuse_dev_do_write()
2040 if (oh.error <= -512 || oh.error > 0) in fuse_dev_do_write()
2043 spin_lock(&fpq->lock); in fuse_dev_do_write()
2045 if (fpq->connected) in fuse_dev_do_write()
2048 err = -ENOENT; in fuse_dev_do_write()
2050 spin_unlock(&fpq->lock); in fuse_dev_do_write()
2057 spin_unlock(&fpq->lock); in fuse_dev_do_write()
2061 err = -EINVAL; in fuse_dev_do_write()
2062 else if (oh.error == -ENOSYS) in fuse_dev_do_write()
2063 fc->no_interrupt = 1; in fuse_dev_do_write()
2064 else if (oh.error == -EAGAIN) in fuse_dev_do_write()
2072 clear_bit(FR_SENT, &req->flags); in fuse_dev_do_write()
2073 list_move(&req->list, &fpq->io); in fuse_dev_do_write()
2074 req->out.h = oh; in fuse_dev_do_write()
2075 set_bit(FR_LOCKED, &req->flags); in fuse_dev_do_write()
2076 spin_unlock(&fpq->lock); in fuse_dev_do_write()
2077 cs->req = req; in fuse_dev_do_write()
2078 if (!req->args->page_replace) in fuse_dev_do_write()
2079 cs->move_pages = 0; in fuse_dev_do_write()
2082 err = nbytes != sizeof(oh) ? -EINVAL : 0; in fuse_dev_do_write()
2084 err = fuse_copy_out_args(cs, req->args, nbytes); in fuse_dev_do_write()
2085 fuse_copy_finish(cs); in fuse_dev_do_write()
2087 spin_lock(&fpq->lock); in fuse_dev_do_write()
2088 clear_bit(FR_LOCKED, &req->flags); in fuse_dev_do_write()
2089 if (!fpq->connected) in fuse_dev_do_write()
2090 err = -ENOENT; in fuse_dev_do_write()
2092 req->out.h.error = -EIO; in fuse_dev_do_write()
2093 if (!test_bit(FR_PRIVATE, &req->flags)) in fuse_dev_do_write()
2094 list_del_init(&req->list); in fuse_dev_do_write()
2095 spin_unlock(&fpq->lock); in fuse_dev_do_write()
2102 fuse_copy_finish(cs); in fuse_dev_do_write()
2108 struct fuse_copy_state cs; in fuse_dev_write() local
2109 struct fuse_dev *fud = fuse_get_dev(iocb->ki_filp); in fuse_dev_write()
2112 return -EPERM; in fuse_dev_write()
2115 return -EINVAL; in fuse_dev_write()
2117 fuse_copy_init(&cs, 0, from); in fuse_dev_write()
2119 return fuse_dev_do_write(fud, &cs, iov_iter_count(from)); in fuse_dev_write()
2130 struct fuse_copy_state cs; in fuse_dev_splice_write() local
2137 return -EPERM; in fuse_dev_splice_write()
2141 head = pipe->head; in fuse_dev_splice_write()
2142 tail = pipe->tail; in fuse_dev_splice_write()
2148 return -ENOMEM; in fuse_dev_splice_write()
2154 rem += pipe_buf(pipe, idx)->len; in fuse_dev_splice_write()
2156 ret = -EINVAL; in fuse_dev_splice_write()
2171 if (rem >= ibuf->len) { in fuse_dev_splice_write()
2173 ibuf->ops = NULL; in fuse_dev_splice_write()
2175 pipe->tail = tail; in fuse_dev_splice_write()
2181 obuf->flags &= ~PIPE_BUF_FLAG_GIFT; in fuse_dev_splice_write()
2182 obuf->len = rem; in fuse_dev_splice_write()
2183 ibuf->offset += obuf->len; in fuse_dev_splice_write()
2184 ibuf->len -= obuf->len; in fuse_dev_splice_write()
2187 rem -= obuf->len; in fuse_dev_splice_write()
2191 fuse_copy_init(&cs, 0, NULL); in fuse_dev_splice_write()
2192 cs.pipebufs = bufs; in fuse_dev_splice_write()
2193 cs.nr_segs = nbuf; in fuse_dev_splice_write()
2194 cs.pipe = pipe; in fuse_dev_splice_write()
2197 cs.move_pages = 1; in fuse_dev_splice_write()
2199 ret = fuse_dev_do_write(fud, &cs, len); in fuse_dev_splice_write()
2206 if (buf->ops) in fuse_dev_splice_write()
2224 fiq = &fud->fc->iq; in fuse_dev_poll()
2225 poll_wait(file, &fiq->waitq, wait); in fuse_dev_poll()
2227 spin_lock(&fiq->lock); in fuse_dev_poll()
2228 if (!fiq->connected) in fuse_dev_poll()
2232 spin_unlock(&fiq->lock); in fuse_dev_poll()
2242 req = list_entry(head->next, struct fuse_req, list); in fuse_dev_end_requests()
2243 req->out.h.error = -ECONNABORTED; in fuse_dev_end_requests()
2244 clear_bit(FR_SENT, &req->flags); in fuse_dev_end_requests()
2245 list_del_init(&req->list); in fuse_dev_end_requests()
2254 p = rb_first(&fc->polled_files); in end_polls()
2259 wake_up_interruptible_all(&ff->poll_wait); in end_polls()
2285 struct fuse_iqueue *fiq = &fc->iq; in fuse_abort_conn()
2287 spin_lock(&fc->lock); in fuse_abort_conn()
2288 if (fc->connected) { in fuse_abort_conn()
2294 /* Background queuing checks fc->connected under bg_lock */ in fuse_abort_conn()
2295 spin_lock(&fc->bg_lock); in fuse_abort_conn()
2296 fc->connected = 0; in fuse_abort_conn()
2297 spin_unlock(&fc->bg_lock); in fuse_abort_conn()
2300 list_for_each_entry(fud, &fc->devices, entry) { in fuse_abort_conn()
2301 struct fuse_pqueue *fpq = &fud->pq; in fuse_abort_conn()
2303 spin_lock(&fpq->lock); in fuse_abort_conn()
2304 fpq->connected = 0; in fuse_abort_conn()
2305 list_for_each_entry_safe(req, next, &fpq->io, list) { in fuse_abort_conn()
2306 req->out.h.error = -ECONNABORTED; in fuse_abort_conn()
2307 spin_lock(&req->waitq.lock); in fuse_abort_conn()
2308 set_bit(FR_ABORTED, &req->flags); in fuse_abort_conn()
2309 if (!test_bit(FR_LOCKED, &req->flags)) { in fuse_abort_conn()
2310 set_bit(FR_PRIVATE, &req->flags); in fuse_abort_conn()
2312 list_move(&req->list, &to_end); in fuse_abort_conn()
2314 spin_unlock(&req->waitq.lock); in fuse_abort_conn()
2317 list_splice_tail_init(&fpq->processing[i], in fuse_abort_conn()
2319 spin_unlock(&fpq->lock); in fuse_abort_conn()
2321 spin_lock(&fc->bg_lock); in fuse_abort_conn()
2322 fc->blocked = 0; in fuse_abort_conn()
2323 fc->max_background = UINT_MAX; in fuse_abort_conn()
2325 spin_unlock(&fc->bg_lock); in fuse_abort_conn()
2327 spin_lock(&fiq->lock); in fuse_abort_conn()
2328 fiq->connected = 0; in fuse_abort_conn()
2329 list_for_each_entry(req, &fiq->pending, list) in fuse_abort_conn()
2330 clear_bit(FR_PENDING, &req->flags); in fuse_abort_conn()
2331 list_splice_tail_init(&fiq->pending, &to_end); in fuse_abort_conn()
2334 wake_up_all(&fiq->waitq); in fuse_abort_conn()
2335 spin_unlock(&fiq->lock); in fuse_abort_conn()
2336 kill_fasync(&fiq->fasync, SIGIO, POLL_IN); in fuse_abort_conn()
2338 wake_up_all(&fc->blocked_waitq); in fuse_abort_conn()
2339 spin_unlock(&fc->lock); in fuse_abort_conn()
2344 * fc->lock must not be taken to avoid conflicts with io-uring in fuse_abort_conn()
2349 spin_unlock(&fc->lock); in fuse_abort_conn()
2358 wait_event(fc->blocked_waitq, atomic_read(&fc->num_waiting) == 0); in fuse_wait_aborted()
2368 struct fuse_conn *fc = fud->fc; in fuse_dev_release()
2369 struct fuse_pqueue *fpq = &fud->pq; in fuse_dev_release()
2373 spin_lock(&fpq->lock); in fuse_dev_release()
2374 WARN_ON(!list_empty(&fpq->io)); in fuse_dev_release()
2376 list_splice_init(&fpq->processing[i], &to_end); in fuse_dev_release()
2377 spin_unlock(&fpq->lock); in fuse_dev_release()
2382 if (atomic_dec_and_test(&fc->dev_count)) { in fuse_dev_release()
2383 WARN_ON(fc->iq.fasync != NULL); in fuse_dev_release()
2397 return -EPERM; in fuse_dev_fasync()
2399 /* No locking - fasync_helper does its own locking */ in fuse_dev_fasync()
2400 return fasync_helper(fd, file, on, &fud->fc->iq.fasync); in fuse_dev_fasync()
2407 if (new->private_data) in fuse_device_clone()
2408 return -EINVAL; in fuse_device_clone()
2412 return -ENOMEM; in fuse_device_clone()
2414 new->private_data = fud; in fuse_device_clone()
2415 atomic_inc(&fc->dev_count); in fuse_device_clone()
2427 return -EFAULT; in fuse_dev_ioctl_clone()
2431 return -EINVAL; in fuse_dev_ioctl_clone()
2434 * Check against file->f_op because CUSE in fuse_dev_ioctl_clone()
2437 if (fd_file(f)->f_op == file->f_op) in fuse_dev_ioctl_clone()
2440 res = -EINVAL; in fuse_dev_ioctl_clone()
2443 res = fuse_device_clone(fud->fc, file); in fuse_dev_ioctl_clone()
2457 return -EPERM; in fuse_dev_ioctl_backing_open()
2460 return -EOPNOTSUPP; in fuse_dev_ioctl_backing_open()
2463 return -EFAULT; in fuse_dev_ioctl_backing_open()
2465 return fuse_backing_open(fud->fc, &map); in fuse_dev_ioctl_backing_open()
2474 return -EPERM; in fuse_dev_ioctl_backing_close()
2477 return -EOPNOTSUPP; in fuse_dev_ioctl_backing_close()
2480 return -EFAULT; in fuse_dev_ioctl_backing_close()
2482 return fuse_backing_close(fud->fc, backing_id); in fuse_dev_ioctl_backing_close()
2501 return -ENOTTY; in fuse_dev_ioctl()
2531 int err = -ENOMEM; in fuse_dev_init()