Lines Matching +full:no +full:- +full:unaligned +full:- +full:direct +full:- +full:access
1 // SPDX-License-Identifier: GPL-2.0-only
3 * linux/fs/nfs/direct.c
7 * High-performance uncached I/O for the Linux NFS client
10 * depends on uncached access to file data. Database clusters
15 * from a local cache. A streaming video server, for instance, has no
21 * correct unaligned requests from applications. All requested bytes are
22 * held on permanent storage before a direct write system call returns to
27 * also supports uncaching whole NFS partitions with "-o forcedirectio,"
33 * 18 Dec 2001 Initial implementation for 2.4 --cel
34 * 08 Jul 2002 Version for 2.4.19, with bug fixes --trondmy
35 * 08 Jun 2003 Port to 2.5 APIs --cel
36 * 31 Mar 2004 Handle direct I/O without VFS support --cel
37 * 15 Sep 2004 Parallel async reads --cel
38 * 04 May 2005 support O_DIRECT with aio --cel
77 atomic_inc(&dreq->io_count); in get_dreq()
82 return atomic_dec_and_test(&dreq->io_count); in put_dreq()
90 if (!(test_bit(NFS_IOHDR_ERROR, &hdr->flags) || in nfs_direct_handle_truncated()
91 test_bit(NFS_IOHDR_EOF, &hdr->flags))) in nfs_direct_handle_truncated()
93 if (dreq->max_count >= dreq_len) { in nfs_direct_handle_truncated()
94 dreq->max_count = dreq_len; in nfs_direct_handle_truncated()
95 if (dreq->count > dreq_len) in nfs_direct_handle_truncated()
96 dreq->count = dreq_len; in nfs_direct_handle_truncated()
99 if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) && !dreq->error) in nfs_direct_handle_truncated()
100 dreq->error = hdr->error; in nfs_direct_handle_truncated()
107 loff_t hdr_end = hdr->io_start + hdr->good_bytes; in nfs_direct_count_bytes()
110 if (hdr_end > dreq->io_start) in nfs_direct_count_bytes()
111 dreq_len = hdr_end - dreq->io_start; in nfs_direct_count_bytes()
115 if (dreq_len > dreq->max_count) in nfs_direct_count_bytes()
116 dreq_len = dreq->max_count; in nfs_direct_count_bytes()
118 if (dreq->count < dreq_len) in nfs_direct_count_bytes()
119 dreq->count = dreq_len; in nfs_direct_count_bytes()
126 size_t req_start = (size_t)(offs - dreq->io_start); in nfs_direct_truncate_request()
128 if (req_start < dreq->max_count) in nfs_direct_truncate_request()
129 dreq->max_count = req_start; in nfs_direct_truncate_request()
130 if (req_start < dreq->count) in nfs_direct_truncate_request()
131 dreq->count = req_start; in nfs_direct_truncate_request()
142 NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_SIZE; in nfs_direct_file_adjust_size_locked()
149 * nfs_swap_rw - NFS address space operation for swap I/O
153 * Perform IO to the swap-file. This is much like direct IO.
178 cinfo->inode = dreq->inode; in nfs_init_cinfo_from_dreq()
179 cinfo->mds = &dreq->mds_cinfo; in nfs_init_cinfo_from_dreq()
180 cinfo->ds = &dreq->ds_cinfo; in nfs_init_cinfo_from_dreq()
181 cinfo->dreq = dreq; in nfs_init_cinfo_from_dreq()
182 cinfo->completion_ops = &nfs_direct_commit_completion_ops; in nfs_init_cinfo_from_dreq()
193 kref_init(&dreq->kref); in nfs_direct_req_alloc()
194 kref_get(&dreq->kref); in nfs_direct_req_alloc()
195 init_completion(&dreq->completion); in nfs_direct_req_alloc()
196 INIT_LIST_HEAD(&dreq->mds_cinfo.list); in nfs_direct_req_alloc()
197 pnfs_init_ds_commit_info(&dreq->ds_cinfo); in nfs_direct_req_alloc()
198 INIT_WORK(&dreq->work, nfs_direct_write_schedule_work); in nfs_direct_req_alloc()
199 spin_lock_init(&dreq->lock); in nfs_direct_req_alloc()
208 pnfs_release_ds_info(&dreq->ds_cinfo, dreq->inode); in nfs_direct_req_free()
209 if (dreq->l_ctx != NULL) in nfs_direct_req_free()
210 nfs_put_lock_context(dreq->l_ctx); in nfs_direct_req_free()
211 if (dreq->ctx != NULL) in nfs_direct_req_free()
212 put_nfs_open_context(dreq->ctx); in nfs_direct_req_free()
218 kref_put(&dreq->kref, nfs_direct_req_free); in nfs_direct_req_release()
223 loff_t start = offset - dreq->io_start; in nfs_dreq_bytes_left()
224 return dreq->max_count - start; in nfs_dreq_bytes_left()
229 * Collects and returns the final error value/byte-count.
233 ssize_t result = -EIOCBQUEUED; in nfs_direct_wait()
236 if (dreq->iocb) in nfs_direct_wait()
239 result = wait_for_completion_killable(&dreq->completion); in nfs_direct_wait()
242 result = dreq->count; in nfs_direct_wait()
243 WARN_ON_ONCE(dreq->count < 0); in nfs_direct_wait()
246 result = dreq->error; in nfs_direct_wait()
253 * Synchronous I/O uses a stack-allocated iocb. Thus we can't trust
258 struct inode *inode = dreq->inode; in nfs_direct_complete()
262 if (dreq->iocb) { in nfs_direct_complete()
263 long res = (long) dreq->error; in nfs_direct_complete()
264 if (dreq->count != 0) { in nfs_direct_complete()
265 res = (long) dreq->count; in nfs_direct_complete()
266 WARN_ON_ONCE(dreq->count < 0); in nfs_direct_complete()
268 dreq->iocb->ki_complete(dreq->iocb, res); in nfs_direct_complete()
271 complete(&dreq->completion); in nfs_direct_complete()
279 struct nfs_direct_req *dreq = hdr->dreq; in nfs_direct_read_completion()
281 spin_lock(&dreq->lock); in nfs_direct_read_completion()
282 if (test_bit(NFS_IOHDR_REDO, &hdr->flags)) { in nfs_direct_read_completion()
283 spin_unlock(&dreq->lock); in nfs_direct_read_completion()
288 spin_unlock(&dreq->lock); in nfs_direct_read_completion()
290 nfs_update_delegated_atime(dreq->inode); in nfs_direct_read_completion()
292 while (!list_empty(&hdr->pages)) { in nfs_direct_read_completion()
293 struct nfs_page *req = nfs_list_entry(hdr->pages.next); in nfs_direct_read_completion()
294 struct page *page = req->wb_page; in nfs_direct_read_completion()
296 if (!PageCompound(page) && bytes < hdr->good_bytes && in nfs_direct_read_completion()
297 (dreq->flags == NFS_ODIRECT_SHOULD_DIRTY)) in nfs_direct_read_completion()
299 bytes += req->wb_bytes; in nfs_direct_read_completion()
306 hdr->release(hdr); in nfs_direct_read_completion()
314 req = nfs_list_entry(head->next); in nfs_read_sync_pgio_error()
322 get_dreq(hdr->dreq); in nfs_direct_pgio_init()
323 set_bit(NFS_IOHDR_ODIRECT, &hdr->flags); in nfs_direct_pgio_init()
337 * no requests have been sent, just return an error.
345 struct inode *inode = dreq->inode; in nfs_direct_read_schedule_iovec()
346 ssize_t result = -EINVAL; in nfs_direct_read_schedule_iovec()
348 size_t rsize = max_t(size_t, NFS_SERVER(inode)->rsize, PAGE_SIZE); in nfs_direct_read_schedule_iovec()
350 nfs_pageio_init_read(&desc, dreq->inode, false, in nfs_direct_read_schedule_iovec()
368 npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE; in nfs_direct_read_schedule_iovec()
371 unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase); in nfs_direct_read_schedule_iovec()
373 req = nfs_page_create_from_page(dreq->ctx, pagevec[i], in nfs_direct_read_schedule_iovec()
385 bytes -= req_len; in nfs_direct_read_schedule_iovec()
398 * If no bytes were started, return the error, and let the in nfs_direct_read_schedule_iovec()
404 return result < 0 ? result : -EIO; in nfs_direct_read_schedule_iovec()
413 * nfs_file_direct_read - file direct read operation for NFS files
418 * We use this function for direct reads instead of calling
421 * to work, we must generate a GETATTR before each direct read, and
435 struct file *file = iocb->ki_filp; in nfs_file_direct_read()
436 struct address_space *mapping = file->f_mapping; in nfs_file_direct_read()
437 struct inode *inode = mapping->host; in nfs_file_direct_read()
442 nfs_add_stats(mapping->host, NFSIOS_DIRECTREADBYTES, count); in nfs_file_direct_read()
444 dfprintk(FILE, "NFS: direct read(%pD2, %zd@%Ld)\n", in nfs_file_direct_read()
445 file, count, (long long) iocb->ki_pos); in nfs_file_direct_read()
453 result = -ENOMEM; in nfs_file_direct_read()
458 dreq->inode = inode; in nfs_file_direct_read()
459 dreq->max_count = count; in nfs_file_direct_read()
460 dreq->io_start = iocb->ki_pos; in nfs_file_direct_read()
461 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp)); in nfs_file_direct_read()
462 l_ctx = nfs_get_lock_context(dreq->ctx); in nfs_file_direct_read()
468 dreq->l_ctx = l_ctx; in nfs_file_direct_read()
470 dreq->iocb = iocb; in nfs_file_direct_read()
473 dreq->flags = NFS_ODIRECT_SHOULD_DIRTY; in nfs_file_direct_read()
486 NFS_I(inode)->read_io += count; in nfs_file_direct_read()
487 requested = nfs_direct_read_schedule_iovec(dreq, iter, iocb->ki_pos); in nfs_file_direct_read()
495 requested -= result; in nfs_file_direct_read()
496 iocb->ki_pos += result; in nfs_file_direct_read()
512 struct nfs_page *head = req->wb_head; in nfs_direct_add_page_head()
514 if (!list_empty(&head->wb_list) || !nfs_lock_request(head)) in nfs_direct_add_page_head()
516 if (!list_empty(&head->wb_list)) { in nfs_direct_add_page_head()
520 list_add(&head->wb_list, list); in nfs_direct_add_page_head()
521 kref_get(&head->wb_kref); in nfs_direct_add_page_head()
522 kref_get(&head->wb_kref); in nfs_direct_add_page_head()
532 if (req->wb_head != req) { in nfs_direct_join_group()
533 nfs_direct_add_page_head(&req->wb_list, req); in nfs_direct_join_group()
536 subreq = req->wb_this_page; in nfs_direct_join_group()
544 if (!list_empty(&subreq->wb_list)) { in nfs_direct_join_group()
548 } while ((subreq = subreq->wb_this_page) != req); in nfs_direct_join_group()
558 mutex_lock(&NFS_I(cinfo->inode)->commit_mutex); in nfs_direct_write_scan_commit_list()
560 nfs_scan_commit_list(&cinfo->mds->list, list, cinfo, 0); in nfs_direct_write_scan_commit_list()
561 mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); in nfs_direct_write_scan_commit_list()
572 nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo); in nfs_direct_write_reschedule()
574 nfs_direct_join_group(&reqs, &cinfo, dreq->inode); in nfs_direct_write_reschedule()
576 nfs_clear_pnfs_ds_commit_verifiers(&dreq->ds_cinfo); in nfs_direct_write_reschedule()
579 nfs_pageio_init_write(&desc, dreq->inode, FLUSH_STABLE, false, in nfs_direct_write_reschedule()
586 req->wb_nio++; in nfs_direct_write_reschedule()
588 spin_lock(&dreq->lock); in nfs_direct_write_reschedule()
589 if (dreq->error < 0) { in nfs_direct_write_reschedule()
590 desc.pg_error = dreq->error; in nfs_direct_write_reschedule()
591 } else if (desc.pg_error != -EAGAIN) { in nfs_direct_write_reschedule()
592 dreq->flags = 0; in nfs_direct_write_reschedule()
594 desc.pg_error = -EIO; in nfs_direct_write_reschedule()
595 dreq->error = desc.pg_error; in nfs_direct_write_reschedule()
597 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_write_reschedule()
598 spin_unlock(&dreq->lock); in nfs_direct_write_reschedule()
609 if (desc.pg_error == -EAGAIN) { in nfs_direct_write_reschedule()
612 spin_lock(&dreq->lock); in nfs_direct_write_reschedule()
614 spin_unlock(&dreq->lock); in nfs_direct_write_reschedule()
625 const struct nfs_writeverf *verf = data->res.verf; in nfs_direct_commit_complete()
626 struct nfs_direct_req *dreq = data->dreq; in nfs_direct_commit_complete()
629 int status = data->task.tk_status; in nfs_direct_commit_complete()
633 spin_lock(&dreq->lock); in nfs_direct_commit_complete()
636 dreq->error = status; in nfs_direct_commit_complete()
637 dreq->flags = NFS_ODIRECT_DONE; in nfs_direct_commit_complete()
639 status = dreq->error; in nfs_direct_commit_complete()
641 spin_unlock(&dreq->lock); in nfs_direct_commit_complete()
645 while (!list_empty(&data->pages)) { in nfs_direct_commit_complete()
646 req = nfs_list_entry(data->pages.next); in nfs_direct_commit_complete()
649 spin_lock(&dreq->lock); in nfs_direct_commit_complete()
651 spin_unlock(&dreq->lock); in nfs_direct_commit_complete()
654 spin_lock(&dreq->lock); in nfs_direct_commit_complete()
655 if (dreq->flags == 0) in nfs_direct_commit_complete()
656 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_commit_complete()
657 spin_unlock(&dreq->lock); in nfs_direct_commit_complete()
662 req->wb_nio = 0; in nfs_direct_commit_complete()
676 struct nfs_direct_req *dreq = cinfo->dreq; in nfs_direct_resched_write()
680 spin_lock(&dreq->lock); in nfs_direct_resched_write()
681 if (dreq->flags != NFS_ODIRECT_DONE) in nfs_direct_resched_write()
682 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_resched_write()
683 spin_unlock(&dreq->lock); in nfs_direct_resched_write()
700 nfs_scan_commit(dreq->inode, &mds_list, &cinfo); in nfs_direct_commit_schedule()
701 res = nfs_generic_commit_list(dreq->inode, &mds_list, 0, &cinfo); in nfs_direct_commit_schedule()
702 if (res < 0) { /* res == -ENOMEM */ in nfs_direct_commit_schedule()
703 spin_lock(&dreq->lock); in nfs_direct_commit_schedule()
704 if (dreq->flags == 0) in nfs_direct_commit_schedule()
705 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_commit_schedule()
706 spin_unlock(&dreq->lock); in nfs_direct_commit_schedule()
719 nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo); in nfs_direct_write_clear_reqs()
733 int flags = dreq->flags; in nfs_direct_write_schedule_work()
735 dreq->flags = 0; in nfs_direct_write_schedule_work()
745 nfs_zap_mapping(dreq->inode, dreq->inode->i_mapping); in nfs_direct_write_schedule_work()
753 queue_work(nfsiod_workqueue, &dreq->work); /* Calls nfs_direct_write_schedule_work */ in nfs_direct_write_complete()
758 struct nfs_direct_req *dreq = hdr->dreq; in nfs_direct_write_completion()
760 struct nfs_page *req = nfs_list_entry(hdr->pages.next); in nfs_direct_write_completion()
761 struct inode *inode = dreq->inode; in nfs_direct_write_completion()
768 spin_lock(&dreq->lock); in nfs_direct_write_completion()
769 if (test_bit(NFS_IOHDR_REDO, &hdr->flags)) { in nfs_direct_write_completion()
770 spin_unlock(&dreq->lock); in nfs_direct_write_completion()
775 if (test_bit(NFS_IOHDR_UNSTABLE_WRITES, &hdr->flags) && in nfs_direct_write_completion()
776 !test_bit(NFS_IOHDR_ERROR, &hdr->flags)) { in nfs_direct_write_completion()
777 if (!dreq->flags) in nfs_direct_write_completion()
778 dreq->flags = NFS_ODIRECT_DO_COMMIT; in nfs_direct_write_completion()
779 flags = dreq->flags; in nfs_direct_write_completion()
781 spin_unlock(&dreq->lock); in nfs_direct_write_completion()
783 spin_lock(&inode->i_lock); in nfs_direct_write_completion()
784 nfs_direct_file_adjust_size_locked(inode, dreq->io_start, dreq->count); in nfs_direct_write_completion()
785 nfs_update_delegated_mtime_locked(dreq->inode); in nfs_direct_write_completion()
786 spin_unlock(&inode->i_lock); in nfs_direct_write_completion()
788 while (!list_empty(&hdr->pages)) { in nfs_direct_write_completion()
790 req = nfs_list_entry(hdr->pages.next); in nfs_direct_write_completion()
793 kref_get(&req->wb_kref); in nfs_direct_write_completion()
794 memcpy(&req->wb_verf, &hdr->verf.verifier, in nfs_direct_write_completion()
795 sizeof(req->wb_verf)); in nfs_direct_write_completion()
796 nfs_mark_request_commit(req, hdr->lseg, &cinfo, in nfs_direct_write_completion()
797 hdr->ds_commit_idx); in nfs_direct_write_completion()
799 kref_get(&req->wb_kref); in nfs_direct_write_completion()
808 hdr->release(hdr); in nfs_direct_write_completion()
816 req = nfs_list_entry(head->next); in nfs_write_sync_pgio_error()
824 struct nfs_direct_req *dreq = hdr->dreq; in nfs_direct_write_reschedule_io()
831 spin_lock(&dreq->lock); in nfs_direct_write_reschedule_io()
832 if (dreq->error == 0) in nfs_direct_write_reschedule_io()
833 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_write_reschedule_io()
834 set_bit(NFS_IOHDR_REDO, &hdr->flags); in nfs_direct_write_reschedule_io()
835 spin_unlock(&dreq->lock); in nfs_direct_write_reschedule_io()
836 while (!list_empty(&hdr->pages)) { in nfs_direct_write_reschedule_io()
837 req = nfs_list_entry(hdr->pages.next); in nfs_direct_write_reschedule_io()
861 * no requests have been sent, just return an error.
868 struct inode *inode = dreq->inode; in nfs_direct_write_schedule_iovec()
872 size_t wsize = max_t(size_t, NFS_SERVER(inode)->wsize, PAGE_SIZE); in nfs_direct_write_schedule_iovec()
883 NFS_I(inode)->write_io += iov_iter_count(iter); in nfs_direct_write_schedule_iovec()
896 npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE; in nfs_direct_write_schedule_iovec()
899 unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase); in nfs_direct_write_schedule_iovec()
901 req = nfs_page_create_from_page(dreq->ctx, pagevec[i], in nfs_direct_write_schedule_iovec()
915 bytes -= req_len; in nfs_direct_write_schedule_iovec()
929 if (desc.pg_error < 0 && desc.pg_error != -EAGAIN) { in nfs_direct_write_schedule_iovec()
937 spin_lock(&dreq->lock); in nfs_direct_write_schedule_iovec()
938 dreq->flags = NFS_ODIRECT_RESCHED_WRITES; in nfs_direct_write_schedule_iovec()
939 spin_unlock(&dreq->lock); in nfs_direct_write_schedule_iovec()
953 * If no bytes were started, return the error, and let the in nfs_direct_write_schedule_iovec()
959 return result < 0 ? result : -EIO; in nfs_direct_write_schedule_iovec()
968 * nfs_file_direct_write - file direct write operation for NFS files
973 * We use this function for direct writes instead of calling
980 * We eliminate local atime updates, see direct read above.
985 * Note that O_APPEND is not supported for NFS direct writes, as there
986 * is no atomic O_APPEND write facility in the NFS protocol.
993 struct file *file = iocb->ki_filp; in nfs_file_direct_write()
994 struct address_space *mapping = file->f_mapping; in nfs_file_direct_write()
995 struct inode *inode = mapping->host; in nfs_file_direct_write()
1000 dfprintk(FILE, "NFS: direct write(%pD2, %zd@%Ld)\n", in nfs_file_direct_write()
1001 file, iov_iter_count(iter), (long long) iocb->ki_pos); in nfs_file_direct_write()
1011 nfs_add_stats(mapping->host, NFSIOS_DIRECTWRITTENBYTES, count); in nfs_file_direct_write()
1013 pos = iocb->ki_pos; in nfs_file_direct_write()
1014 end = (pos + iov_iter_count(iter) - 1) >> PAGE_SHIFT; in nfs_file_direct_write()
1018 result = -ENOMEM; in nfs_file_direct_write()
1023 dreq->inode = inode; in nfs_file_direct_write()
1024 dreq->max_count = count; in nfs_file_direct_write()
1025 dreq->io_start = pos; in nfs_file_direct_write()
1026 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp)); in nfs_file_direct_write()
1027 l_ctx = nfs_get_lock_context(dreq->ctx); in nfs_file_direct_write()
1033 dreq->l_ctx = l_ctx; in nfs_file_direct_write()
1035 dreq->iocb = iocb; in nfs_file_direct_write()
1036 pnfs_init_ds_commit_info_ops(&dreq->ds_cinfo, inode); in nfs_file_direct_write()
1054 if (mapping->nrpages) { in nfs_file_direct_write()
1065 requested -= result; in nfs_file_direct_write()
1066 iocb->ki_pos = pos + result; in nfs_file_direct_write()
1082 * nfs_init_directcache - create a slab cache for nfs_direct_req structures
1092 return -ENOMEM; in nfs_init_directcache()
1098 * nfs_destroy_directcache - destroy the slab cache for nfs_direct_req structures