109 		struct io_mapped_ubuf *imu = node->buf;  in io_buffer_unmap()  local
111 		if (!refcount_dec_and_test(&imu->refs))  in io_buffer_unmap()
113 		for (i = 0; i < imu->nr_bvecs; i++)  in io_buffer_unmap()
114 			unpin_user_page(imu->bvec[i].bv_page);  in io_buffer_unmap()
115 		if (imu->acct_pages)  in io_buffer_unmap()
116 			io_unaccount_mem(ctx, imu->acct_pages);  in io_buffer_unmap()
117 		kvfree(imu);  in io_buffer_unmap()
592 		struct io_mapped_ubuf *imu;  in headpage_already_acct()  local
596 		imu = node->buf;  in headpage_already_acct()
597 		for (j = 0; j < imu->nr_bvecs; j++) {  in headpage_already_acct()
598 			if (!PageCompound(imu->bvec[j].bv_page))  in headpage_already_acct()
600 			if (compound_head(imu->bvec[j].bv_page) == hpage)  in headpage_already_acct()
609 				 int nr_pages, struct io_mapped_ubuf *imu,  in io_buffer_account_pin()  argument
614 	imu->acct_pages = 0;  in io_buffer_account_pin()
617 			imu->acct_pages++;  in io_buffer_account_pin()
627 			imu->acct_pages += page_size(hpage) >> PAGE_SHIFT;  in io_buffer_account_pin()
631 	if (!imu->acct_pages)  in io_buffer_account_pin()
634 	ret = io_account_mem(ctx, imu->acct_pages);  in io_buffer_account_pin()
636 		imu->acct_pages = 0;  in io_buffer_account_pin()
732 	struct io_mapped_ubuf *imu = NULL;  in io_sqe_buffer_register()  local
764 	imu = kvmalloc(struct_size(imu, bvec, nr_pages), GFP_KERNEL);  in io_sqe_buffer_register()
765 	if (!imu)  in io_sqe_buffer_register()
768 	ret = io_buffer_account_pin(ctx, pages, nr_pages, imu, last_hpage);  in io_sqe_buffer_register()
776 	imu->ubuf = (unsigned long) iov->iov_base;  in io_sqe_buffer_register()
777 	imu->len = iov->iov_len;  in io_sqe_buffer_register()
778 	imu->nr_bvecs = nr_pages;  in io_sqe_buffer_register()
779 	imu->folio_shift = PAGE_SHIFT;  in io_sqe_buffer_register()
781 		imu->folio_shift = data.folio_shift;  in io_sqe_buffer_register()
782 	refcount_set(&imu->refs, 1);  in io_sqe_buffer_register()
783 	off = (unsigned long) iov->iov_base & ((1UL << imu->folio_shift) - 1);  in io_sqe_buffer_register()
784 	node->buf = imu;  in io_sqe_buffer_register()
790 		vec_len = min_t(size_t, size, (1UL << imu->folio_shift) - off);  in io_sqe_buffer_register()
791 		bvec_set_page(&imu->bvec[i], pages[i], vec_len, off);  in io_sqe_buffer_register()
797 		kvfree(imu);  in io_sqe_buffer_register()
879 			   struct io_mapped_ubuf *imu,  in io_import_fixed()  argument
885 	if (WARN_ON_ONCE(!imu))  in io_import_fixed()
890 	if (unlikely(buf_addr < imu->ubuf || buf_end > (imu->ubuf + imu->len)))  in io_import_fixed()
897 	offset = buf_addr - imu->ubuf;  in io_import_fixed()
898 	iov_iter_bvec(iter, ddir, imu->bvec, imu->nr_bvecs, len);  in io_import_fixed()
917 		const struct bio_vec *bvec = imu->bvec;  in io_import_fixed()
926 			seg_skip = 1 + (offset >> imu->folio_shift);  in io_import_fixed()
930 			iter->iov_offset = offset & ((1UL << imu->folio_shift) - 1);  in io_import_fixed()