347 	struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);  in sio_write_complete()  local
348 	struct page *page = sio->bvec[0].bv_page;  in sio_write_complete()
351 	if (ret != sio->len) {  in sio_write_complete()
362 		for (p = 0; p < sio->pages; p++) {  in sio_write_complete()
363 			page = sio->bvec[p].bv_page;  in sio_write_complete()
369 	for (p = 0; p < sio->pages; p++)  in sio_write_complete()
370 		end_page_writeback(sio->bvec[p].bv_page);  in sio_write_complete()
372 	mempool_free(sio, sio_pool);  in sio_write_complete()
377 	struct swap_iocb *sio = NULL;  in swap_writepage_fs()  local
386 		sio = *wbc->swap_plug;  in swap_writepage_fs()
387 	if (sio) {  in swap_writepage_fs()
388 		if (sio->iocb.ki_filp != swap_file ||  in swap_writepage_fs()
389 		    sio->iocb.ki_pos + sio->len != pos) {  in swap_writepage_fs()
390 			swap_write_unplug(sio);  in swap_writepage_fs()
391 			sio = NULL;  in swap_writepage_fs()
394 	if (!sio) {  in swap_writepage_fs()
395 		sio = mempool_alloc(sio_pool, GFP_NOIO);  in swap_writepage_fs()
396 		init_sync_kiocb(&sio->iocb, swap_file);  in swap_writepage_fs()
397 		sio->iocb.ki_complete = sio_write_complete;  in swap_writepage_fs()
398 		sio->iocb.ki_pos = pos;  in swap_writepage_fs()
399 		sio->pages = 0;  in swap_writepage_fs()
400 		sio->len = 0;  in swap_writepage_fs()
402 	bvec_set_folio(&sio->bvec[sio->pages], folio, folio_size(folio), 0);  in swap_writepage_fs()
403 	sio->len += folio_size(folio);  in swap_writepage_fs()
404 	sio->pages += 1;  in swap_writepage_fs()
405 	if (sio->pages == ARRAY_SIZE(sio->bvec) || !wbc->swap_plug) {  in swap_writepage_fs()
406 		swap_write_unplug(sio);  in swap_writepage_fs()
407 		sio = NULL;  in swap_writepage_fs()
410 		*wbc->swap_plug = sio;  in swap_writepage_fs()
476 void swap_write_unplug(struct swap_iocb *sio)  in swap_write_unplug()  argument
479 	struct address_space *mapping = sio->iocb.ki_filp->f_mapping;  in swap_write_unplug()
482 	iov_iter_bvec(&from, ITER_SOURCE, sio->bvec, sio->pages, sio->len);  in swap_write_unplug()
483 	ret = mapping->a_ops->swap_rw(&sio->iocb, &from);  in swap_write_unplug()
485 		sio_write_complete(&sio->iocb, ret);  in swap_write_unplug()
490 	struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);  in sio_read_complete()  local
493 	if (ret == sio->len) {  in sio_read_complete()
494 		for (p = 0; p < sio->pages; p++) {  in sio_read_complete()
495 			struct folio *folio = page_folio(sio->bvec[p].bv_page);  in sio_read_complete()
502 		count_vm_events(PSWPIN, sio->pages);  in sio_read_complete()
504 		for (p = 0; p < sio->pages; p++) {  in sio_read_complete()
505 			struct folio *folio = page_folio(sio->bvec[p].bv_page);  in sio_read_complete()
511 	mempool_free(sio, sio_pool);  in sio_read_complete()
547 	struct swap_iocb *sio = NULL;  in swap_read_folio_fs()  local
551 		sio = *plug;  in swap_read_folio_fs()
552 	if (sio) {  in swap_read_folio_fs()
553 		if (sio->iocb.ki_filp != sis->swap_file ||  in swap_read_folio_fs()
554 		    sio->iocb.ki_pos + sio->len != pos) {  in swap_read_folio_fs()
555 			swap_read_unplug(sio);  in swap_read_folio_fs()
556 			sio = NULL;  in swap_read_folio_fs()
559 	if (!sio) {  in swap_read_folio_fs()
560 		sio = mempool_alloc(sio_pool, GFP_KERNEL);  in swap_read_folio_fs()
561 		init_sync_kiocb(&sio->iocb, sis->swap_file);  in swap_read_folio_fs()
562 		sio->iocb.ki_pos = pos;  in swap_read_folio_fs()
563 		sio->iocb.ki_complete = sio_read_complete;  in swap_read_folio_fs()
564 		sio->pages = 0;  in swap_read_folio_fs()
565 		sio->len = 0;  in swap_read_folio_fs()
567 	bvec_set_folio(&sio->bvec[sio->pages], folio, folio_size(folio), 0);  in swap_read_folio_fs()
568 	sio->len += folio_size(folio);  in swap_read_folio_fs()
569 	sio->pages += 1;  in swap_read_folio_fs()
570 	if (sio->pages == ARRAY_SIZE(sio->bvec) || !plug) {  in swap_read_folio_fs()
571 		swap_read_unplug(sio);  in swap_read_folio_fs()
572 		sio = NULL;  in swap_read_folio_fs()
575 		*plug = sio;  in swap_read_folio_fs()
665 void __swap_read_unplug(struct swap_iocb *sio)  in __swap_read_unplug()  argument
668 	struct address_space *mapping = sio->iocb.ki_filp->f_mapping;  in __swap_read_unplug()
671 	iov_iter_bvec(&from, ITER_DEST, sio->bvec, sio->pages, sio->len);  in __swap_read_unplug()
672 	ret = mapping->a_ops->swap_rw(&sio->iocb, &from);  in __swap_read_unplug()
674 		sio_read_complete(&sio->iocb, ret);  in __swap_read_unplug()