Lines Matching +full:fiq +full:- +full:device
1 // SPDX-License-Identifier: GPL-2.0
3 * virtio-fs: Virtio Filesystem
31 /* List of virtio-fs device instances and a lock for the list. Also provides
32 * mutual exclusion in device removal and mounting path
47 /* Per-virtqueue state */
50 struct virtqueue *vq; /* protected by ->lock */
63 /* A virtio-fs device instance */
125 struct fuse_fs_context *ctx = fsc->fs_private; in virtio_fs_parse_param()
134 ctx->dax_mode = FUSE_DAX_ALWAYS; in virtio_fs_parse_param()
137 ctx->dax_mode = result.uint_32; in virtio_fs_parse_param()
140 return -EINVAL; in virtio_fs_parse_param()
148 struct fuse_fs_context *ctx = fsc->fs_private; in virtio_fs_free_fsc()
155 struct virtio_fs *fs = vq->vdev->priv; in vq_to_fsvq()
157 return &fs->vqs[vq->index]; in vq_to_fsvq()
160 /* Should be called with fsvq->lock held. */
163 fsvq->in_flight++; in inc_in_flight_req()
166 /* Should be called with fsvq->lock held. */
169 WARN_ON(fsvq->in_flight <= 0); in dec_in_flight_req()
170 fsvq->in_flight--; in dec_in_flight_req()
171 if (!fsvq->in_flight) in dec_in_flight_req()
172 complete(&fsvq->in_flight_zero); in dec_in_flight_req()
180 return sysfs_emit(buf, "%s\n", fs->tag); in tag_show()
195 kfree(vfs->mq_map); in virtio_fs_ktype_release()
196 kfree(vfs->vqs); in virtio_fs_ktype_release()
211 for (i = 0; i < fs->nvqs; i++) { in virtio_fs_kobj_to_vq()
212 if (kobj == fs->vqs[i].kobj) in virtio_fs_kobj_to_vq()
213 return &fs->vqs[i]; in virtio_fs_kobj_to_vq()
221 struct virtio_fs *fs = container_of(kobj->parent->parent, struct virtio_fs, kobj); in name_show()
225 return -EINVAL; in name_show()
226 return sysfs_emit(buf, "%s\n", fsvq->name); in name_show()
234 struct virtio_fs *fs = container_of(kobj->parent->parent, struct virtio_fs, kobj); in cpu_list_show()
237 const size_t size = PAGE_SIZE - 1; in cpu_list_show()
242 return -EINVAL; in cpu_list_show()
244 qid = fsvq->vq->index; in cpu_list_show()
246 if (qid < VQ_REQUEST || (fs->mq_map[cpu] == qid)) { in cpu_list_show()
248 ret = snprintf(buf + pos, size - pos, "%u", cpu); in cpu_list_show()
250 ret = snprintf(buf + pos, size - pos, ", %u", cpu); in cpu_list_show()
252 if (ret >= size - pos) in cpu_list_show()
258 ret = snprintf(buf + pos, size + 1 - pos, "\n"); in cpu_list_show()
279 kobject_put(&fs->kobj); in virtio_fs_put_locked()
289 static void virtio_fs_fiq_release(struct fuse_iqueue *fiq) in virtio_fs_fiq_release() argument
291 struct virtio_fs *vfs = fiq->priv; in virtio_fs_fiq_release()
298 WARN_ON(fsvq->in_flight < 0); in virtio_fs_drain_queue()
301 spin_lock(&fsvq->lock); in virtio_fs_drain_queue()
302 if (fsvq->in_flight) { in virtio_fs_drain_queue()
306 reinit_completion(&fsvq->in_flight_zero); in virtio_fs_drain_queue()
307 spin_unlock(&fsvq->lock); in virtio_fs_drain_queue()
308 wait_for_completion(&fsvq->in_flight_zero); in virtio_fs_drain_queue()
310 spin_unlock(&fsvq->lock); in virtio_fs_drain_queue()
313 flush_work(&fsvq->done_work); in virtio_fs_drain_queue()
314 flush_work(&fsvq->dispatch_work); in virtio_fs_drain_queue()
322 for (i = 0; i < fs->nvqs; i++) { in virtio_fs_drain_all_queues_locked()
323 fsvq = &fs->vqs[i]; in virtio_fs_drain_all_queues_locked()
330 /* Provides mutual exclusion between ->remove and ->kill_sb in virtio_fs_drain_all_queues()
346 for (i = 0; i < fs->nvqs; i++) { in virtio_fs_start_all_queues()
347 fsvq = &fs->vqs[i]; in virtio_fs_start_all_queues()
348 spin_lock(&fsvq->lock); in virtio_fs_start_all_queues()
349 fsvq->connected = true; in virtio_fs_start_all_queues()
350 spin_unlock(&fsvq->lock); in virtio_fs_start_all_queues()
359 for (i = 0; i < fs->nvqs; i++) { in virtio_fs_delete_queues_sysfs()
360 fsvq = &fs->vqs[i]; in virtio_fs_delete_queues_sysfs()
361 kobject_put(fsvq->kobj); in virtio_fs_delete_queues_sysfs()
371 for (i = 0; i < fs->nvqs; i++) { in virtio_fs_add_queues_sysfs()
372 fsvq = &fs->vqs[i]; in virtio_fs_add_queues_sysfs()
375 fsvq->kobj = kobject_create_and_add(buff, fs->mqs_kobj); in virtio_fs_add_queues_sysfs()
376 if (!fs->mqs_kobj) { in virtio_fs_add_queues_sysfs()
377 ret = -ENOMEM; in virtio_fs_add_queues_sysfs()
381 ret = sysfs_create_group(fsvq->kobj, &virtio_fs_vq_attr_group); in virtio_fs_add_queues_sysfs()
383 kobject_put(fsvq->kobj); in virtio_fs_add_queues_sysfs()
392 fsvq = &fs->vqs[j]; in virtio_fs_add_queues_sysfs()
393 kobject_put(fsvq->kobj); in virtio_fs_add_queues_sysfs()
398 /* Add a new instance to the list or return -EEXIST if tag name exists*/
408 if (strcmp(fs->tag, fs2->tag) == 0) { in virtio_fs_add_instance()
410 return -EEXIST; in virtio_fs_add_instance()
419 fs->kobj.kset = virtio_fs_kset; in virtio_fs_add_instance()
420 ret = kobject_add(&fs->kobj, NULL, "%d", vdev->index); in virtio_fs_add_instance()
424 fs->mqs_kobj = kobject_create_and_add("mqs", &fs->kobj); in virtio_fs_add_instance()
425 if (!fs->mqs_kobj) { in virtio_fs_add_instance()
426 ret = -ENOMEM; in virtio_fs_add_instance()
430 ret = sysfs_create_link(&fs->kobj, &vdev->dev.kobj, "device"); in virtio_fs_add_instance()
438 list_add_tail(&fs->list, &virtio_fs_instances); in virtio_fs_add_instance()
442 kobject_uevent(&fs->kobj, KOBJ_ADD); in virtio_fs_add_instance()
447 sysfs_remove_link(&fs->kobj, "device"); in virtio_fs_add_instance()
449 kobject_put(fs->mqs_kobj); in virtio_fs_add_instance()
451 kobject_del(&fs->kobj); in virtio_fs_add_instance()
465 if (strcmp(fs->tag, tag) == 0) { in virtio_fs_find_instance()
466 kobject_get(&fs->kobj); in virtio_fs_find_instance()
483 for (i = 0; i < fs->nvqs; i++) { in virtio_fs_free_devs()
484 struct virtio_fs_vq *fsvq = &fs->vqs[i]; in virtio_fs_free_devs()
486 if (!fsvq->fud) in virtio_fs_free_devs()
489 fuse_dev_free(fsvq->fud); in virtio_fs_free_devs()
490 fsvq->fud = NULL; in virtio_fs_free_devs()
494 /* Read filesystem name from virtio config into fs->tag (must kfree()). */
505 return -EINVAL; /* empty tag */ in virtio_fs_read_tag()
509 len = end - tag_buf; in virtio_fs_read_tag()
510 fs->tag = devm_kmalloc(&vdev->dev, len + 1, GFP_KERNEL); in virtio_fs_read_tag()
511 if (!fs->tag) in virtio_fs_read_tag()
512 return -ENOMEM; in virtio_fs_read_tag()
513 memcpy(fs->tag, tag_buf, len); in virtio_fs_read_tag()
514 fs->tag[len] = '\0'; in virtio_fs_read_tag()
517 * awkward on mount(8) command-lines and cause problems in the sysfs in virtio_fs_read_tag()
520 if (strchr(fs->tag, '\n')) { in virtio_fs_read_tag()
521 dev_dbg(&vdev->dev, "refusing virtiofs tag with newline character\n"); in virtio_fs_read_tag()
522 return -EINVAL; in virtio_fs_read_tag()
525 dev_info(&vdev->dev, "discovered new tag: %s\n", fs->tag); in virtio_fs_read_tag()
534 struct virtqueue *vq = fsvq->vq; in virtio_fs_hiprio_done_work()
537 spin_lock(&fsvq->lock); in virtio_fs_hiprio_done_work()
550 if (!list_empty(&fsvq->queued_reqs)) in virtio_fs_hiprio_done_work()
551 schedule_work(&fsvq->dispatch_work); in virtio_fs_hiprio_done_work()
553 spin_unlock(&fsvq->lock); in virtio_fs_hiprio_done_work()
563 pr_debug("virtio-fs: worker %s called.\n", __func__); in virtio_fs_request_dispatch_work()
565 spin_lock(&fsvq->lock); in virtio_fs_request_dispatch_work()
566 req = list_first_entry_or_null(&fsvq->end_reqs, struct fuse_req, in virtio_fs_request_dispatch_work()
569 spin_unlock(&fsvq->lock); in virtio_fs_request_dispatch_work()
573 list_del_init(&req->list); in virtio_fs_request_dispatch_work()
574 spin_unlock(&fsvq->lock); in virtio_fs_request_dispatch_work()
582 spin_lock(&fsvq->lock); in virtio_fs_request_dispatch_work()
583 req = list_first_entry_or_null(&fsvq->queued_reqs, in virtio_fs_request_dispatch_work()
586 spin_unlock(&fsvq->lock); in virtio_fs_request_dispatch_work()
589 list_del_init(&req->list); in virtio_fs_request_dispatch_work()
590 spin_unlock(&fsvq->lock); in virtio_fs_request_dispatch_work()
596 if (ret == -ENOSPC) { in virtio_fs_request_dispatch_work()
597 spin_lock(&fsvq->lock); in virtio_fs_request_dispatch_work()
598 list_add_tail(&req->list, &fsvq->queued_reqs); in virtio_fs_request_dispatch_work()
599 spin_unlock(&fsvq->lock); in virtio_fs_request_dispatch_work()
602 req->out.h.error = ret; in virtio_fs_request_dispatch_work()
603 spin_lock(&fsvq->lock); in virtio_fs_request_dispatch_work()
605 spin_unlock(&fsvq->lock); in virtio_fs_request_dispatch_work()
606 pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n", in virtio_fs_request_dispatch_work()
625 struct virtio_fs_forget_req *req = &forget->req; in send_forget_request()
627 spin_lock(&fsvq->lock); in send_forget_request()
628 if (!fsvq->connected) { in send_forget_request()
636 vq = fsvq->vq; in send_forget_request()
637 dev_dbg(&vq->vdev->dev, "%s\n", __func__); in send_forget_request()
641 if (ret == -ENOSPC) { in send_forget_request()
642 pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later\n", in send_forget_request()
644 list_add_tail(&forget->list, &fsvq->queued_reqs); in send_forget_request()
650 pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n", in send_forget_request()
662 spin_unlock(&fsvq->lock); in send_forget_request()
668 spin_unlock(&fsvq->lock); in send_forget_request()
677 pr_debug("virtio-fs: worker %s called.\n", __func__); in virtio_fs_hiprio_dispatch_work()
679 spin_lock(&fsvq->lock); in virtio_fs_hiprio_dispatch_work()
680 forget = list_first_entry_or_null(&fsvq->queued_reqs, in virtio_fs_hiprio_dispatch_work()
683 spin_unlock(&fsvq->lock); in virtio_fs_hiprio_dispatch_work()
687 list_del(&forget->list); in virtio_fs_hiprio_dispatch_work()
688 spin_unlock(&fsvq->lock); in virtio_fs_hiprio_dispatch_work()
694 /* Allocate and copy args into req->argbuf */
697 struct fuse_args *args = req->args; in copy_args_to_argbuf()
704 num_in = args->in_numargs - args->in_pages; in copy_args_to_argbuf()
705 num_out = args->out_numargs - args->out_pages; in copy_args_to_argbuf()
706 len = fuse_len_args(num_in, (struct fuse_arg *) args->in_args) + in copy_args_to_argbuf()
707 fuse_len_args(num_out, args->out_args); in copy_args_to_argbuf()
709 req->argbuf = kmalloc(len, gfp); in copy_args_to_argbuf()
710 if (!req->argbuf) in copy_args_to_argbuf()
711 return -ENOMEM; in copy_args_to_argbuf()
714 memcpy(req->argbuf + offset, in copy_args_to_argbuf()
715 args->in_args[i].value, in copy_args_to_argbuf()
716 args->in_args[i].size); in copy_args_to_argbuf()
717 offset += args->in_args[i].size; in copy_args_to_argbuf()
723 /* Copy args out of and free req->argbuf */
732 remaining = req->out.h.len - sizeof(req->out.h); in copy_args_from_argbuf()
733 num_in = args->in_numargs - args->in_pages; in copy_args_from_argbuf()
734 num_out = args->out_numargs - args->out_pages; in copy_args_from_argbuf()
735 offset = fuse_len_args(num_in, (struct fuse_arg *)args->in_args); in copy_args_from_argbuf()
738 unsigned int argsize = args->out_args[i].size; in copy_args_from_argbuf()
740 if (args->out_argvar && in copy_args_from_argbuf()
741 i == args->out_numargs - 1 && in copy_args_from_argbuf()
746 memcpy(args->out_args[i].value, req->argbuf + offset, argsize); in copy_args_from_argbuf()
749 if (i != args->out_numargs - 1) in copy_args_from_argbuf()
750 remaining -= argsize; in copy_args_from_argbuf()
753 /* Store the actual size of the variable-length arg */ in copy_args_from_argbuf()
754 if (args->out_argvar) in copy_args_from_argbuf()
755 args->out_args[args->out_numargs - 1].size = remaining; in copy_args_from_argbuf()
757 kfree(req->argbuf); in copy_args_from_argbuf()
758 req->argbuf = NULL; in copy_args_from_argbuf()
765 struct fuse_pqueue *fpq = &fsvq->fud->pq; in virtio_fs_request_complete()
775 args = req->args; in virtio_fs_request_complete()
778 if (args->out_pages && args->page_zeroing) { in virtio_fs_request_complete()
779 len = args->out_args[args->out_numargs - 1].size; in virtio_fs_request_complete()
781 for (i = 0; i < ap->num_folios; i++) { in virtio_fs_request_complete()
782 thislen = ap->descs[i].length; in virtio_fs_request_complete()
784 WARN_ON(ap->descs[i].offset); in virtio_fs_request_complete()
785 folio = ap->folios[i]; in virtio_fs_request_complete()
789 len -= thislen; in virtio_fs_request_complete()
794 spin_lock(&fpq->lock); in virtio_fs_request_complete()
795 clear_bit(FR_SENT, &req->flags); in virtio_fs_request_complete()
796 spin_unlock(&fpq->lock); in virtio_fs_request_complete()
799 spin_lock(&fsvq->lock); in virtio_fs_request_complete()
801 spin_unlock(&fsvq->lock); in virtio_fs_request_complete()
809 virtio_fs_request_complete(w->req, w->fsvq); in virtio_fs_complete_req_work()
817 struct fuse_pqueue *fpq = &fsvq->fud->pq; in virtio_fs_requests_done_work()
818 struct virtqueue *vq = fsvq->vq; in virtio_fs_requests_done_work()
825 spin_lock(&fsvq->lock); in virtio_fs_requests_done_work()
830 spin_lock(&fpq->lock); in virtio_fs_requests_done_work()
831 list_move_tail(&req->list, &reqs); in virtio_fs_requests_done_work()
832 spin_unlock(&fpq->lock); in virtio_fs_requests_done_work()
835 spin_unlock(&fsvq->lock); in virtio_fs_requests_done_work()
839 list_del_init(&req->list); in virtio_fs_requests_done_work()
842 if (req->args->may_block) { in virtio_fs_requests_done_work()
846 INIT_WORK(&w->done_work, virtio_fs_complete_req_work); in virtio_fs_requests_done_work()
847 w->fsvq = fsvq; in virtio_fs_requests_done_work()
848 w->req = req; in virtio_fs_requests_done_work()
849 schedule_work(&w->done_work); in virtio_fs_requests_done_work()
856 spin_lock(&fsvq->lock); in virtio_fs_requests_done_work()
857 if (!list_empty(&fsvq->queued_reqs)) in virtio_fs_requests_done_work()
858 schedule_work(&fsvq->dispatch_work); in virtio_fs_requests_done_work()
859 spin_unlock(&fsvq->lock); in virtio_fs_requests_done_work()
868 * e.g. PCIe MSI-X in virtio_fs_map_queues()
870 if (!vdev->config->get_vq_affinity) in virtio_fs_map_queues()
873 for (q = 0; q < fs->num_request_queues; q++) { in virtio_fs_map_queues()
874 mask = vdev->config->get_vq_affinity(vdev, VQ_REQUEST + q); in virtio_fs_map_queues()
879 fs->mq_map[cpu] = q + VQ_REQUEST; in virtio_fs_map_queues()
885 masks = group_cpus_evenly(fs->num_request_queues); in virtio_fs_map_queues()
889 fs->mq_map[cpu] = VQ_REQUEST; in virtio_fs_map_queues()
893 for (q = 0; q < fs->num_request_queues; q++) { in virtio_fs_map_queues()
895 fs->mq_map[cpu] = q + VQ_REQUEST; in virtio_fs_map_queues()
905 dev_dbg(&vq->vdev->dev, "%s %s\n", __func__, fsvq->name); in virtio_fs_vq_done()
907 schedule_work(&fsvq->done_work); in virtio_fs_vq_done()
913 strscpy(fsvq->name, name, VQ_NAME_LEN); in virtio_fs_init_vq()
914 spin_lock_init(&fsvq->lock); in virtio_fs_init_vq()
915 INIT_LIST_HEAD(&fsvq->queued_reqs); in virtio_fs_init_vq()
916 INIT_LIST_HEAD(&fsvq->end_reqs); in virtio_fs_init_vq()
917 init_completion(&fsvq->in_flight_zero); in virtio_fs_init_vq()
920 INIT_WORK(&fsvq->done_work, virtio_fs_requests_done_work); in virtio_fs_init_vq()
921 INIT_WORK(&fsvq->dispatch_work, in virtio_fs_init_vq()
924 INIT_WORK(&fsvq->done_work, virtio_fs_hiprio_done_work); in virtio_fs_init_vq()
925 INIT_WORK(&fsvq->dispatch_work, in virtio_fs_init_vq()
938 * the multi-queue mapping and interrupt affinities in virtio_fs_setup_vqs()
945 &fs->num_request_queues); in virtio_fs_setup_vqs()
946 if (fs->num_request_queues == 0) in virtio_fs_setup_vqs()
947 return -EINVAL; in virtio_fs_setup_vqs()
950 fs->num_request_queues = min_t(unsigned int, fs->num_request_queues, in virtio_fs_setup_vqs()
952 fs->nvqs = VQ_REQUEST + fs->num_request_queues; in virtio_fs_setup_vqs()
953 fs->vqs = kcalloc(fs->nvqs, sizeof(fs->vqs[VQ_HIPRIO]), GFP_KERNEL); in virtio_fs_setup_vqs()
954 if (!fs->vqs) in virtio_fs_setup_vqs()
955 return -ENOMEM; in virtio_fs_setup_vqs()
957 vqs = kmalloc_array(fs->nvqs, sizeof(vqs[VQ_HIPRIO]), GFP_KERNEL); in virtio_fs_setup_vqs()
958 fs->mq_map = kcalloc_node(nr_cpu_ids, sizeof(*fs->mq_map), GFP_KERNEL, in virtio_fs_setup_vqs()
959 dev_to_node(&vdev->dev)); in virtio_fs_setup_vqs()
960 vqs_info = kcalloc(fs->nvqs, sizeof(*vqs_info), GFP_KERNEL); in virtio_fs_setup_vqs()
961 if (!vqs || !vqs_info || !fs->mq_map) { in virtio_fs_setup_vqs()
962 ret = -ENOMEM; in virtio_fs_setup_vqs()
968 virtio_fs_init_vq(&fs->vqs[VQ_HIPRIO], "hiprio", VQ_HIPRIO); in virtio_fs_setup_vqs()
969 vqs_info[VQ_HIPRIO].name = fs->vqs[VQ_HIPRIO].name; in virtio_fs_setup_vqs()
972 for (i = VQ_REQUEST; i < fs->nvqs; i++) { in virtio_fs_setup_vqs()
975 snprintf(vq_name, VQ_NAME_LEN, "requests.%u", i - VQ_REQUEST); in virtio_fs_setup_vqs()
976 virtio_fs_init_vq(&fs->vqs[i], vq_name, VQ_REQUEST); in virtio_fs_setup_vqs()
978 vqs_info[i].name = fs->vqs[i].name; in virtio_fs_setup_vqs()
981 ret = virtio_find_vqs(vdev, fs->nvqs, vqs, vqs_info, &desc); in virtio_fs_setup_vqs()
985 for (i = 0; i < fs->nvqs; i++) in virtio_fs_setup_vqs()
986 fs->vqs[i].vq = vqs[i]; in virtio_fs_setup_vqs()
993 kfree(fs->vqs); in virtio_fs_setup_vqs()
994 kfree(fs->mq_map); in virtio_fs_setup_vqs()
999 /* Free virtqueues (device must already be reset) */
1002 vdev->config->del_vqs(vdev); in virtio_fs_cleanup_vqs()
1015 size_t max_nr_pages = fs->window_len / PAGE_SIZE - pgoff; in virtio_fs_direct_access()
1018 *kaddr = fs->window_kaddr + offset; in virtio_fs_direct_access()
1020 *pfn = phys_to_pfn_t(fs->window_phys_addr + offset, in virtio_fs_direct_access()
1069 return rc == -EOPNOTSUPP ? 0 : rc; in virtio_fs_setup_dax()
1076 dev_notice(&vdev->dev, "%s: No cache capability\n", __func__); in virtio_fs_setup_dax()
1080 if (!devm_request_mem_region(&vdev->dev, cache_reg.addr, cache_reg.len, in virtio_fs_setup_dax()
1081 dev_name(&vdev->dev))) { in virtio_fs_setup_dax()
1082 dev_warn(&vdev->dev, "could not reserve region addr=0x%llx len=0x%llx\n", in virtio_fs_setup_dax()
1084 return -EBUSY; in virtio_fs_setup_dax()
1087 dev_notice(&vdev->dev, "Cache len: 0x%llx @ 0x%llx\n", cache_reg.len, in virtio_fs_setup_dax()
1090 pgmap = devm_kzalloc(&vdev->dev, sizeof(*pgmap), GFP_KERNEL); in virtio_fs_setup_dax()
1092 return -ENOMEM; in virtio_fs_setup_dax()
1094 pgmap->type = MEMORY_DEVICE_FS_DAX; in virtio_fs_setup_dax()
1101 pgmap->range = (struct range) { in virtio_fs_setup_dax()
1103 .end = (phys_addr_t) cache_reg.addr + cache_reg.len - 1, in virtio_fs_setup_dax()
1105 pgmap->nr_range = 1; in virtio_fs_setup_dax()
1107 fs->window_kaddr = devm_memremap_pages(&vdev->dev, pgmap); in virtio_fs_setup_dax()
1108 if (IS_ERR(fs->window_kaddr)) in virtio_fs_setup_dax()
1109 return PTR_ERR(fs->window_kaddr); in virtio_fs_setup_dax()
1111 fs->window_phys_addr = (phys_addr_t) cache_reg.addr; in virtio_fs_setup_dax()
1112 fs->window_len = (phys_addr_t) cache_reg.len; in virtio_fs_setup_dax()
1114 dev_dbg(&vdev->dev, "%s: window kaddr 0x%px phys_addr 0x%llx len 0x%llx\n", in virtio_fs_setup_dax()
1115 __func__, fs->window_kaddr, cache_reg.addr, cache_reg.len); in virtio_fs_setup_dax()
1117 fs->dax_dev = no_free_ptr(dax_dev); in virtio_fs_setup_dax()
1118 return devm_add_action_or_reset(&vdev->dev, virtio_fs_cleanup_dax, in virtio_fs_setup_dax()
1119 fs->dax_dev); in virtio_fs_setup_dax()
1129 return -ENOMEM; in virtio_fs_probe()
1130 kobject_init(&fs->kobj, &virtio_fs_ktype); in virtio_fs_probe()
1131 vdev->priv = fs; in virtio_fs_probe()
1147 /* Bring the device online in case the filesystem is mounted and in virtio_fs_probe()
1163 vdev->priv = NULL; in virtio_fs_probe()
1164 kobject_put(&fs->kobj); in virtio_fs_probe()
1173 for (i = 0; i < fs->nvqs; i++) { in virtio_fs_stop_all_queues()
1174 fsvq = &fs->vqs[i]; in virtio_fs_stop_all_queues()
1175 spin_lock(&fsvq->lock); in virtio_fs_stop_all_queues()
1176 fsvq->connected = false; in virtio_fs_stop_all_queues()
1177 spin_unlock(&fsvq->lock); in virtio_fs_stop_all_queues()
1183 struct virtio_fs *fs = vdev->priv; in virtio_fs_remove()
1186 /* This device is going away. No one should get new reference */ in virtio_fs_remove()
1187 list_del_init(&fs->list); in virtio_fs_remove()
1189 sysfs_remove_link(&fs->kobj, "device"); in virtio_fs_remove()
1190 kobject_put(fs->mqs_kobj); in virtio_fs_remove()
1191 kobject_del(&fs->kobj); in virtio_fs_remove()
1197 vdev->priv = NULL; in virtio_fs_remove()
1198 /* Put device reference on virtio_fs object */ in virtio_fs_remove()
1207 pr_warn("virtio-fs: suspend/resume not yet supported\n"); in virtio_fs_freeze()
1208 return -EOPNOTSUPP; in virtio_fs_freeze()
1238 static void virtio_fs_send_forget(struct fuse_iqueue *fiq, struct fuse_forget_link *link) in virtio_fs_send_forget() argument
1242 struct virtio_fs *fs = fiq->priv; in virtio_fs_send_forget()
1243 struct virtio_fs_vq *fsvq = &fs->vqs[VQ_HIPRIO]; in virtio_fs_send_forget()
1244 u64 unique = fuse_get_unique(fiq); in virtio_fs_send_forget()
1248 req = &forget->req; in virtio_fs_send_forget()
1250 req->ih = (struct fuse_in_header){ in virtio_fs_send_forget()
1252 .nodeid = link->forget_one.nodeid, in virtio_fs_send_forget()
1256 req->arg = (struct fuse_forget_in){ in virtio_fs_send_forget()
1257 .nlookup = link->forget_one.nlookup, in virtio_fs_send_forget()
1264 static void virtio_fs_send_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req) in virtio_fs_send_interrupt() argument
1275 /* Count number of scatter-gather elements required */
1285 total_len -= this_len; in sg_count_fuse_folios()
1291 /* Return the number of scatter-gather list elements required */
1294 struct fuse_args *args = req->args; in sg_count_fuse_req()
1298 if (args->in_numargs - args->in_pages) in sg_count_fuse_req()
1301 if (args->in_pages) { in sg_count_fuse_req()
1302 size = args->in_args[args->in_numargs - 1].size; in sg_count_fuse_req()
1303 total_sgs += sg_count_fuse_folios(ap->descs, ap->num_folios, in sg_count_fuse_req()
1307 if (!test_bit(FR_ISREPLY, &req->flags)) in sg_count_fuse_req()
1312 if (args->out_numargs - args->out_pages) in sg_count_fuse_req()
1315 if (args->out_pages) { in sg_count_fuse_req()
1316 size = args->out_args[args->out_numargs - 1].size; in sg_count_fuse_req()
1317 total_sgs += sg_count_fuse_folios(ap->descs, ap->num_folios, in sg_count_fuse_req()
1324 /* Add folios to scatter-gather list and return number of elements used */
1338 total_len -= this_len; in sg_init_fuse_folios()
1344 /* Add args to scatter-gather list and return number of elements used */
1353 struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args); in sg_init_fuse_args()
1357 len = fuse_len_args(numargs - argpages, args); in sg_init_fuse_args()
1363 ap->folios, ap->descs, in sg_init_fuse_args()
1364 ap->num_folios, in sg_init_fuse_args()
1365 args[numargs - 1].size); in sg_init_fuse_args()
1373 /* Add a request to a virtqueue and kick the device */
1384 struct fuse_args *args = req->args; in virtio_fs_enqueue_req()
1400 ret = -ENOMEM; in virtio_fs_enqueue_req()
1411 sg_init_one(&sg[out_sgs++], &req->in.h, sizeof(req->in.h)); in virtio_fs_enqueue_req()
1413 (struct fuse_arg *)args->in_args, in virtio_fs_enqueue_req()
1414 args->in_numargs, args->in_pages, in virtio_fs_enqueue_req()
1415 req->argbuf, &argbuf_used); in virtio_fs_enqueue_req()
1418 if (test_bit(FR_ISREPLY, &req->flags)) { in virtio_fs_enqueue_req()
1420 &req->out.h, sizeof(req->out.h)); in virtio_fs_enqueue_req()
1422 args->out_args, args->out_numargs, in virtio_fs_enqueue_req()
1423 args->out_pages, in virtio_fs_enqueue_req()
1424 req->argbuf + argbuf_used, NULL); in virtio_fs_enqueue_req()
1432 spin_lock(&fsvq->lock); in virtio_fs_enqueue_req()
1434 if (!fsvq->connected) { in virtio_fs_enqueue_req()
1435 spin_unlock(&fsvq->lock); in virtio_fs_enqueue_req()
1436 ret = -ENOTCONN; in virtio_fs_enqueue_req()
1440 vq = fsvq->vq; in virtio_fs_enqueue_req()
1443 spin_unlock(&fsvq->lock); in virtio_fs_enqueue_req()
1448 fpq = &fsvq->fud->pq; in virtio_fs_enqueue_req()
1449 spin_lock(&fpq->lock); in virtio_fs_enqueue_req()
1450 list_add_tail(&req->list, fpq->processing); in virtio_fs_enqueue_req()
1451 spin_unlock(&fpq->lock); in virtio_fs_enqueue_req()
1452 set_bit(FR_SENT, &req->flags); in virtio_fs_enqueue_req()
1460 spin_unlock(&fsvq->lock); in virtio_fs_enqueue_req()
1466 if (ret < 0 && req->argbuf) { in virtio_fs_enqueue_req()
1467 kfree(req->argbuf); in virtio_fs_enqueue_req()
1468 req->argbuf = NULL; in virtio_fs_enqueue_req()
1478 static void virtio_fs_send_req(struct fuse_iqueue *fiq, struct fuse_req *req) in virtio_fs_send_req() argument
1485 if (req->in.h.opcode != FUSE_NOTIFY_REPLY) in virtio_fs_send_req()
1486 req->in.h.unique = fuse_get_unique(fiq); in virtio_fs_send_req()
1488 clear_bit(FR_PENDING, &req->flags); in virtio_fs_send_req()
1490 fs = fiq->priv; in virtio_fs_send_req()
1491 queue_id = fs->mq_map[raw_smp_processor_id()]; in virtio_fs_send_req()
1494 __func__, req->in.h.opcode, req->in.h.unique, in virtio_fs_send_req()
1495 req->in.h.nodeid, req->in.h.len, in virtio_fs_send_req()
1496 fuse_len_args(req->args->out_numargs, req->args->out_args), in virtio_fs_send_req()
1499 fsvq = &fs->vqs[queue_id]; in virtio_fs_send_req()
1502 if (ret == -ENOSPC) { in virtio_fs_send_req()
1505 * context as we might be holding fc->bg_lock. in virtio_fs_send_req()
1507 spin_lock(&fsvq->lock); in virtio_fs_send_req()
1508 list_add_tail(&req->list, &fsvq->queued_reqs); in virtio_fs_send_req()
1510 spin_unlock(&fsvq->lock); in virtio_fs_send_req()
1513 req->out.h.error = ret; in virtio_fs_send_req()
1514 pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n", ret); in virtio_fs_send_req()
1517 spin_lock(&fsvq->lock); in virtio_fs_send_req()
1518 list_add_tail(&req->list, &fsvq->end_reqs); in virtio_fs_send_req()
1519 schedule_work(&fsvq->dispatch_work); in virtio_fs_send_req()
1520 spin_unlock(&fsvq->lock); in virtio_fs_send_req()
1534 ctx->rootmode = S_IFDIR; in virtio_fs_ctx_set_defaults()
1535 ctx->default_permissions = 1; in virtio_fs_ctx_set_defaults()
1536 ctx->allow_other = 1; in virtio_fs_ctx_set_defaults()
1537 ctx->max_read = UINT_MAX; in virtio_fs_ctx_set_defaults()
1538 ctx->blksize = 512; in virtio_fs_ctx_set_defaults()
1539 ctx->destroy = true; in virtio_fs_ctx_set_defaults()
1540 ctx->no_control = true; in virtio_fs_ctx_set_defaults()
1541 ctx->no_force_umount = true; in virtio_fs_ctx_set_defaults()
1547 struct fuse_conn *fc = fm->fc; in virtio_fs_fill_super()
1548 struct virtio_fs *fs = fc->iq.priv; in virtio_fs_fill_super()
1549 struct fuse_fs_context *ctx = fsc->fs_private; in virtio_fs_fill_super()
1556 /* After holding mutex, make sure virtiofs device is still there. in virtio_fs_fill_super()
1557 * Though we are holding a reference to it, drive ->remove might in virtio_fs_fill_super()
1560 err = -EINVAL; in virtio_fs_fill_super()
1561 if (list_empty(&fs->list)) { in virtio_fs_fill_super()
1562 pr_info("virtio-fs: tag <%s> not found\n", fs->tag); in virtio_fs_fill_super()
1566 err = -ENOMEM; in virtio_fs_fill_super()
1568 for (i = 0; i < fs->nvqs; i++) { in virtio_fs_fill_super()
1569 struct virtio_fs_vq *fsvq = &fs->vqs[i]; in virtio_fs_fill_super()
1571 fsvq->fud = fuse_dev_alloc(); in virtio_fs_fill_super()
1572 if (!fsvq->fud) in virtio_fs_fill_super()
1577 ctx->fudptr = NULL; in virtio_fs_fill_super()
1578 if (ctx->dax_mode != FUSE_DAX_NEVER) { in virtio_fs_fill_super()
1579 if (ctx->dax_mode == FUSE_DAX_ALWAYS && !fs->dax_dev) { in virtio_fs_fill_super()
1580 err = -EINVAL; in virtio_fs_fill_super()
1581 pr_err("virtio-fs: dax can't be enabled as filesystem" in virtio_fs_fill_super()
1582 " device does not support it.\n"); in virtio_fs_fill_super()
1585 ctx->dax_dev = fs->dax_dev; in virtio_fs_fill_super()
1591 for (i = 0; i < fs->nvqs; i++) { in virtio_fs_fill_super()
1592 struct virtio_fs_vq *fsvq = &fs->vqs[i]; in virtio_fs_fill_super()
1594 fuse_dev_install(fsvq->fud, fc); in virtio_fs_fill_super()
1612 struct fuse_conn *fc = fm->fc; in virtio_fs_conn_destroy()
1613 struct virtio_fs *vfs = fc->iq.priv; in virtio_fs_conn_destroy()
1614 struct virtio_fs_vq *fsvq = &vfs->vqs[VQ_HIPRIO]; in virtio_fs_conn_destroy()
1623 spin_lock(&fsvq->lock); in virtio_fs_conn_destroy()
1624 fsvq->connected = false; in virtio_fs_conn_destroy()
1625 spin_unlock(&fsvq->lock); in virtio_fs_conn_destroy()
1646 if (sb->s_root) { in virtio_kill_sb()
1658 struct fuse_mount *fsc_fm = fsc->s_fs_info; in virtio_fs_test_super()
1661 return fsc_fm->fc->iq.priv == sb_fm->fc->iq.priv; in virtio_fs_test_super()
1671 int err = -EIO; in virtio_fs_get_tree()
1673 if (!fsc->source) in virtio_fs_get_tree()
1677 * in fc->iq->priv. Once fuse_conn is going away, it calls ->put() in virtio_fs_get_tree()
1680 fs = virtio_fs_find_instance(fsc->source); in virtio_fs_get_tree()
1682 pr_info("virtio-fs: tag <%s> not found\n", fsc->source); in virtio_fs_get_tree()
1683 return -EINVAL; in virtio_fs_get_tree()
1686 virtqueue_size = virtqueue_get_vring_size(fs->vqs[VQ_REQUEST].vq); in virtio_fs_get_tree()
1690 err = -ENOMEM; in virtio_fs_get_tree()
1699 fuse_conn_init(fc, fm, fsc->user_ns, &virtio_fs_fiq_ops, fs); in virtio_fs_get_tree()
1700 fc->release = fuse_free_conn; in virtio_fs_get_tree()
1701 fc->delete_stale = true; in virtio_fs_get_tree()
1702 fc->auto_submounts = true; in virtio_fs_get_tree()
1703 fc->sync_fs = true; in virtio_fs_get_tree()
1704 fc->use_pages_for_kvec_io = true; in virtio_fs_get_tree()
1707 fc->max_pages_limit = min_t(unsigned int, fc->max_pages_limit, in virtio_fs_get_tree()
1708 virtqueue_size - FUSE_HEADER_OVERHEAD); in virtio_fs_get_tree()
1710 fsc->s_fs_info = fm; in virtio_fs_get_tree()
1712 if (fsc->s_fs_info) in virtio_fs_get_tree()
1717 if (!sb->s_root) { in virtio_fs_get_tree()
1724 sb->s_flags |= SB_ACTIVE; in virtio_fs_get_tree()
1727 WARN_ON(fsc->root); in virtio_fs_get_tree()
1728 fsc->root = dget(sb->s_root); in virtio_fs_get_tree()
1747 if (fsc->purpose == FS_CONTEXT_FOR_SUBMOUNT) in virtio_fs_init_fs_context()
1752 return -ENOMEM; in virtio_fs_init_fs_context()
1753 fsc->fs_private = ctx; in virtio_fs_init_fs_context()
1754 fsc->ops = &virtio_fs_context_ops; in virtio_fs_init_fs_context()
1770 add_uevent_var(env, "TAG=%s", fs->tag); in virtio_fs_uevent()
1783 return -ENOMEM; in virtio_fs_sysfs_init()