1 // SPDX-License-Identifier: GPL-2.0-only
20 #define BUFFERED	0x08	/* The pagecache needs cleaning up */  macro
31 	WARN_ON_ONCE(subreq->len - subreq->transferred != iov_iter_count(&subreq->io_iter));  in netfs_clear_unread()
32 	iov_iter_zero(iov_iter_count(&subreq->io_iter), &subreq->io_iter);  in netfs_clear_unread()
33 	if (subreq->start + subreq->transferred >= subreq->rreq->i_size)  in netfs_clear_unread()
34 		__set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags);  in netfs_clear_unread()
49 	if (unlikely(folio_pos(folio) < rreq->abandon_to)) {  in netfs_unlock_read_folio()
57 	if (!test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) {  in netfs_unlock_read_folio()
61 			if (finfo->netfs_group)  in netfs_unlock_read_folio()
62 				folio_change_private(folio, finfo->netfs_group);  in netfs_unlock_read_folio()
68 		if (test_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &rreq->flags)) {  in netfs_unlock_read_folio()
81 		if (test_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &rreq->flags))  in netfs_unlock_read_folio()
86 	if (!test_bit(NETFS_RREQ_DONT_UNLOCK_FOLIOS, &rreq->flags)) {  in netfs_unlock_read_folio()
87 		if (folio->index == rreq->no_unlock_folio &&  in netfs_unlock_read_folio()
88 		    test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags)) {  in netfs_unlock_read_folio()
105 	struct folio_queue *folioq = rreq->buffer.tail;  in netfs_read_unlock_folios()
106 	unsigned long long collected_to = rreq->collected_to;  in netfs_read_unlock_folios()
107 	unsigned int slot = rreq->buffer.first_tail_slot;  in netfs_read_unlock_folios()
109 	if (rreq->cleaned_to >= rreq->collected_to)  in netfs_read_unlock_folios()
115 		folioq = rolling_buffer_delete_spent(&rreq->buffer);  in netfs_read_unlock_folios()
117 			rreq->front_folio_order = 0;  in netfs_read_unlock_folios()
130 			set_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &rreq->flags);  in netfs_read_unlock_folios()
135 			      rreq->debug_id, folio->index))  in netfs_read_unlock_folios()
139 		rreq->front_folio_order = order;  in netfs_read_unlock_folios()
142 		fend = umin(fpos + fsize, rreq->i_size);  in netfs_read_unlock_folios()
151 		WRITE_ONCE(rreq->cleaned_to, fpos + fsize);  in netfs_read_unlock_folios()
154 		clear_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &rreq->flags);  in netfs_read_unlock_folios()
163 			folioq = rolling_buffer_delete_spent(&rreq->buffer);  in netfs_read_unlock_folios()
174 	rreq->buffer.tail = folioq;  in netfs_read_unlock_folios()
176 	rreq->buffer.first_tail_slot = slot;  in netfs_read_unlock_folios()
190 	struct netfs_io_stream *stream = &rreq->io_streams[0];  in netfs_collect_read_results()
193 	_enter("%llx-%llx", rreq->start, rreq->start + rreq->len);  in netfs_collect_read_results()
198 	if (rreq->origin == NETFS_READAHEAD ||  in netfs_collect_read_results()
199 	    rreq->origin == NETFS_READPAGE ||  in netfs_collect_read_results()
200 	    rreq->origin == NETFS_READ_FOR_WRITE)  in netfs_collect_read_results()
201 		notes = BUFFERED;  in netfs_collect_read_results()
210 	front = READ_ONCE(stream->front);  in netfs_collect_read_results()
216 		       front->debug_index, front->start, front->transferred, front->len);  in netfs_collect_read_results()
218 		if (stream->collected_to < front->start) {  in netfs_collect_read_results()
219 			trace_netfs_collect_gap(rreq, stream, front->start, 'F');  in netfs_collect_read_results()
220 			stream->collected_to = front->start;  in netfs_collect_read_results()
223 		if (test_bit(NETFS_SREQ_IN_PROGRESS, &front->flags))  in netfs_collect_read_results()
226 		transferred = READ_ONCE(front->transferred);  in netfs_collect_read_results()
233 		if (notes & BUFFERED) {  in netfs_collect_read_results()
234 			size_t fsize = PAGE_SIZE << rreq->front_folio_order;  in netfs_collect_read_results()
238 			    front->error == 0 &&  in netfs_collect_read_results()
239 			    transferred < front->len &&  in netfs_collect_read_results()
240 			    (test_bit(NETFS_SREQ_HIT_EOF, &front->flags) ||  in netfs_collect_read_results()
241 			     test_bit(NETFS_SREQ_CLEAR_TAIL, &front->flags))) {  in netfs_collect_read_results()
243 				transferred = front->transferred = front->len;  in netfs_collect_read_results()
247 			stream->collected_to = front->start + transferred;  in netfs_collect_read_results()
248 			rreq->collected_to = stream->collected_to;  in netfs_collect_read_results()
250 			if (test_bit(NETFS_SREQ_COPY_TO_CACHE, &front->flags))  in netfs_collect_read_results()
253 			if (test_bit(NETFS_SREQ_FAILED, &front->flags)) {  in netfs_collect_read_results()
254 				rreq->abandon_to = front->start + front->len;  in netfs_collect_read_results()
255 				front->transferred = front->len;  in netfs_collect_read_results()
256 				transferred = front->len;  in netfs_collect_read_results()
259 			if (front->start + transferred >= rreq->cleaned_to + fsize ||  in netfs_collect_read_results()
260 			    test_bit(NETFS_SREQ_HIT_EOF, &front->flags))  in netfs_collect_read_results()
263 			stream->collected_to = front->start + transferred;  in netfs_collect_read_results()
264 			rreq->collected_to = stream->collected_to;  in netfs_collect_read_results()
271 		if (test_bit(NETFS_SREQ_FAILED, &front->flags)) {  in netfs_collect_read_results()
272 			if (!stream->failed) {  in netfs_collect_read_results()
273 				stream->error = front->error;  in netfs_collect_read_results()
274 				rreq->error = front->error;  in netfs_collect_read_results()
275 				set_bit(NETFS_RREQ_FAILED, &rreq->flags);  in netfs_collect_read_results()
276 				stream->failed = true;  in netfs_collect_read_results()
279 		} else if (test_bit(NETFS_SREQ_NEED_RETRY, &front->flags)) {  in netfs_collect_read_results()
280 			stream->need_retry = true;  in netfs_collect_read_results()
284 			if (!stream->failed)  in netfs_collect_read_results()
285 				stream->transferred = stream->collected_to - rreq->start;  in netfs_collect_read_results()
290 		stream->source = front->source;  in netfs_collect_read_results()
291 		spin_lock(&rreq->lock);  in netfs_collect_read_results()
295 		list_del_init(&front->rreq_link);  in netfs_collect_read_results()
296 		front = list_first_entry_or_null(&stream->subrequests,  in netfs_collect_read_results()
298 		stream->front = front;  in netfs_collect_read_results()
299 		spin_unlock(&rreq->lock);  in netfs_collect_read_results()
307 	trace_netfs_collect_state(rreq, rreq->collected_to, notes);  in netfs_collect_read_results()
309 	if (!(notes & BUFFERED))  in netfs_collect_read_results()
310 		rreq->cleaned_to = rreq->collected_to;  in netfs_collect_read_results()
314 	if ((notes & MADE_PROGRESS) && test_bit(NETFS_RREQ_PAUSE, &rreq->flags)) {  in netfs_collect_read_results()
316 		clear_bit_unlock(NETFS_RREQ_PAUSE, &rreq->flags);  in netfs_collect_read_results()
318 		wake_up(&rreq->waitq);  in netfs_collect_read_results()
346 	struct netfs_io_stream *stream = &rreq->io_streams[0];  in netfs_rreq_assess_dio()
352 	list_for_each_entry(subreq, &stream->subrequests, rreq_link) {  in netfs_rreq_assess_dio()
353 		rreq->transferred += subreq->transferred;  in netfs_rreq_assess_dio()
355 		if (subreq->transferred < subreq->len ||  in netfs_rreq_assess_dio()
356 		    test_bit(NETFS_SREQ_FAILED, &subreq->flags)) {  in netfs_rreq_assess_dio()
357 			rreq->error = subreq->error;  in netfs_rreq_assess_dio()
362 	if (rreq->origin == NETFS_DIO_READ) {  in netfs_rreq_assess_dio()
363 		for (i = 0; i < rreq->direct_bv_count; i++) {  in netfs_rreq_assess_dio()
364 			flush_dcache_page(rreq->direct_bv[i].bv_page);  in netfs_rreq_assess_dio()
368 			set_page_dirty(rreq->direct_bv[i].bv_page);  in netfs_rreq_assess_dio()
372 	if (rreq->iocb) {  in netfs_rreq_assess_dio()
373 		rreq->iocb->ki_pos += rreq->transferred;  in netfs_rreq_assess_dio()
374 		if (rreq->iocb->ki_complete)  in netfs_rreq_assess_dio()
375 			rreq->iocb->ki_complete(  in netfs_rreq_assess_dio()
376 				rreq->iocb, rreq->error ? rreq->error : rreq->transferred);  in netfs_rreq_assess_dio()
378 	if (rreq->netfs_ops->done)  in netfs_rreq_assess_dio()
379 		rreq->netfs_ops->done(rreq);  in netfs_rreq_assess_dio()
380 	if (rreq->origin == NETFS_DIO_READ)  in netfs_rreq_assess_dio()
381 		inode_dio_end(rreq->inode);  in netfs_rreq_assess_dio()
389 	struct netfs_io_stream *stream = &rreq->io_streams[0];  in netfs_rreq_assess_single()
391 	if (!rreq->error && stream->source == NETFS_DOWNLOAD_FROM_SERVER &&  in netfs_rreq_assess_single()
392 	    fscache_resources_valid(&rreq->cache_resources)) {  in netfs_rreq_assess_single()
394 		netfs_single_mark_inode_dirty(rreq->inode);  in netfs_rreq_assess_single()
397 	if (rreq->iocb) {  in netfs_rreq_assess_single()
398 		rreq->iocb->ki_pos += rreq->transferred;  in netfs_rreq_assess_single()
399 		if (rreq->iocb->ki_complete)  in netfs_rreq_assess_single()
400 			rreq->iocb->ki_complete(  in netfs_rreq_assess_single()
401 				rreq->iocb, rreq->error ? rreq->error : rreq->transferred);  in netfs_rreq_assess_single()
403 	if (rreq->netfs_ops->done)  in netfs_rreq_assess_single()
404 		rreq->netfs_ops->done(rreq);  in netfs_rreq_assess_single()
415 	struct netfs_io_stream *stream = &rreq->io_streams[0];  in netfs_read_collection()
422 	if (!test_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags))  in netfs_read_collection()
426 	if (!list_empty(&stream->subrequests))  in netfs_read_collection()
430 	rreq->transferred = stream->transferred;  in netfs_read_collection()
435 	switch (rreq->origin) {  in netfs_read_collection()
446 	task_io_account_read(rreq->transferred);  in netfs_read_collection()
449 	clear_and_wake_up_bit(NETFS_RREQ_IN_PROGRESS, &rreq->flags);  in netfs_read_collection()
454 	if (unlikely(rreq->copy_to_cache))  in netfs_read_collection()
463 	if (test_bit(NETFS_RREQ_IN_PROGRESS, &rreq->flags))  in netfs_read_collection_worker()
473 	if (test_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &rreq->flags) &&  in netfs_wake_read_collector()
474 	    !test_bit(NETFS_RREQ_RETRYING, &rreq->flags)) {  in netfs_wake_read_collector()
475 		if (!work_pending(&rreq->work)) {  in netfs_wake_read_collector()
477 			if (!queue_work(system_unbound_wq, &rreq->work))  in netfs_wake_read_collector()
482 		wake_up(&rreq->waitq);  in netfs_wake_read_collector()
487  * netfs_read_subreq_progress - Note progress of a read operation.
493  * Before calling, the filesystem should update subreq->transferred to track
498 	struct netfs_io_request *rreq = subreq->rreq;  in netfs_read_subreq_progress()
499 	struct netfs_io_stream *stream = &rreq->io_streams[0];  in netfs_read_subreq_progress()
500 	size_t fsize = PAGE_SIZE << rreq->front_folio_order;  in netfs_read_subreq_progress()
507 	if (subreq->start + subreq->transferred > rreq->cleaned_to + fsize &&  in netfs_read_subreq_progress()
508 	    (rreq->origin == NETFS_READAHEAD ||  in netfs_read_subreq_progress()
509 	     rreq->origin == NETFS_READPAGE ||  in netfs_read_subreq_progress()
510 	     rreq->origin == NETFS_READ_FOR_WRITE) &&  in netfs_read_subreq_progress()
511 	    list_is_first(&subreq->rreq_link, &stream->subrequests)  in netfs_read_subreq_progress()
513 		__set_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);  in netfs_read_subreq_progress()
520  * netfs_read_subreq_terminated - Note the termination of an I/O operation.
526  * The caller indicates the outcome of the operation through @subreq->error,
528  * NETFS_SREQ_NEED_RETRY is set) or a negative error code.  The helper will
532  * Before calling, the filesystem should update subreq->transferred to track
537 	struct netfs_io_request *rreq = subreq->rreq;  in netfs_read_subreq_terminated()
538 	struct netfs_io_stream *stream = &rreq->io_streams[0];  in netfs_read_subreq_terminated()
540 	switch (subreq->source) {  in netfs_read_subreq_terminated()
554 	if (!subreq->error && subreq->transferred < subreq->len) {  in netfs_read_subreq_terminated()
555 		if (test_bit(NETFS_SREQ_HIT_EOF, &subreq->flags)) {  in netfs_read_subreq_terminated()
557 		} else if (test_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags)) {  in netfs_read_subreq_terminated()
559 		} else if (test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) {  in netfs_read_subreq_terminated()
561 		} else if (test_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags)) {  in netfs_read_subreq_terminated()
562 			__set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);  in netfs_read_subreq_terminated()
565 			__set_bit(NETFS_SREQ_FAILED, &subreq->flags);  in netfs_read_subreq_terminated()
566 			subreq->error = -ENODATA;  in netfs_read_subreq_terminated()
571 	if (unlikely(subreq->error < 0)) {  in netfs_read_subreq_terminated()
572 		trace_netfs_failure(rreq, subreq, subreq->error, netfs_fail_read);  in netfs_read_subreq_terminated()
573 		if (subreq->source == NETFS_READ_FROM_CACHE) {  in netfs_read_subreq_terminated()
575 			__set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);  in netfs_read_subreq_terminated()
578 			__set_bit(NETFS_SREQ_FAILED, &subreq->flags);  in netfs_read_subreq_terminated()
581 		set_bit(NETFS_RREQ_PAUSE, &rreq->flags);  in netfs_read_subreq_terminated()
586 	clear_bit_unlock(NETFS_SREQ_IN_PROGRESS, &subreq->flags);  in netfs_read_subreq_terminated()
590 	if (list_is_first(&subreq->rreq_link, &stream->subrequests) ||  in netfs_read_subreq_terminated()
591 	    test_bit(NETFS_RREQ_RETRYING, &rreq->flags))  in netfs_read_subreq_terminated()
606 		subreq->error = 0;  in netfs_cache_read_terminated()
608 			subreq->transferred += transferred_or_error;  in netfs_cache_read_terminated()
609 			__set_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);  in netfs_cache_read_terminated()
612 		subreq->error = transferred_or_error;  in netfs_cache_read_terminated()
623 	struct netfs_io_stream *stream = &rreq->io_streams[0];  in netfs_wait_for_read()
629 		prepare_to_wait(&rreq->waitq, &myself, TASK_UNINTERRUPTIBLE);  in netfs_wait_for_read()
631 		subreq = list_first_entry_or_null(&stream->subrequests,  in netfs_wait_for_read()
634 		    (!test_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags) ||  in netfs_wait_for_read()
635 		     test_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags))) {  in netfs_wait_for_read()
641 		if (!test_bit(NETFS_RREQ_IN_PROGRESS, &rreq->flags))  in netfs_wait_for_read()
648 	finish_wait(&rreq->waitq, &myself);  in netfs_wait_for_read()
650 	ret = rreq->error;  in netfs_wait_for_read()
652 		ret = rreq->transferred;  in netfs_wait_for_read()
653 		switch (rreq->origin) {  in netfs_wait_for_read()
656 			ret = rreq->transferred;  in netfs_wait_for_read()
659 			if (rreq->submitted < rreq->len) {  in netfs_wait_for_read()
661 				ret = -EIO;  in netfs_wait_for_read()
676 	struct netfs_io_stream *stream = &rreq->io_streams[0];  in netfs_wait_for_pause()
683 		prepare_to_wait(&rreq->waitq, &myself, TASK_UNINTERRUPTIBLE);  in netfs_wait_for_pause()
685 		if (!test_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &rreq->flags)) {  in netfs_wait_for_pause()
686 			subreq = list_first_entry_or_null(&stream->subrequests,  in netfs_wait_for_pause()
689 			    (!test_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags) ||  in netfs_wait_for_pause()
690 			     test_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags))) {  in netfs_wait_for_pause()
697 		if (!test_bit(NETFS_RREQ_IN_PROGRESS, &rreq->flags) ||  in netfs_wait_for_pause()
698 		    !test_bit(NETFS_RREQ_PAUSE, &rreq->flags))  in netfs_wait_for_pause()
705 	finish_wait(&rreq->waitq, &myself);  in netfs_wait_for_pause()