1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Copyright (C) 2024, Alibaba Cloud
4 */
5 #include "internal.h"
6 #include <trace/events/erofs.h>
7
8 struct erofs_fileio_rq {
9 struct bio_vec bvecs[16];
10 struct bio bio;
11 struct kiocb iocb;
12 struct super_block *sb;
13 };
14
15 struct erofs_fileio {
16 struct erofs_map_blocks map;
17 struct erofs_map_dev dev;
18 struct erofs_fileio_rq *rq;
19 };
20
erofs_fileio_ki_complete(struct kiocb * iocb,long ret)21 static void erofs_fileio_ki_complete(struct kiocb *iocb, long ret)
22 {
23 struct erofs_fileio_rq *rq =
24 container_of(iocb, struct erofs_fileio_rq, iocb);
25 struct folio_iter fi;
26
27 if (ret > 0) {
28 if (ret != rq->bio.bi_iter.bi_size) {
29 bio_advance(&rq->bio, ret);
30 zero_fill_bio(&rq->bio);
31 }
32 ret = 0;
33 }
34 if (rq->bio.bi_end_io) {
35 if (ret < 0 && !rq->bio.bi_status)
36 rq->bio.bi_status = errno_to_blk_status(ret);
37 rq->bio.bi_end_io(&rq->bio);
38 } else {
39 bio_for_each_folio_all(fi, &rq->bio) {
40 DBG_BUGON(folio_test_uptodate(fi.folio));
41 erofs_onlinefolio_end(fi.folio, ret);
42 }
43 }
44 bio_uninit(&rq->bio);
45 kfree(rq);
46 }
47
erofs_fileio_rq_submit(struct erofs_fileio_rq * rq)48 static void erofs_fileio_rq_submit(struct erofs_fileio_rq *rq)
49 {
50 struct iov_iter iter;
51 int ret;
52
53 if (!rq)
54 return;
55 rq->iocb.ki_pos = rq->bio.bi_iter.bi_sector << SECTOR_SHIFT;
56 rq->iocb.ki_ioprio = get_current_ioprio();
57 rq->iocb.ki_complete = erofs_fileio_ki_complete;
58 if (test_opt(&EROFS_SB(rq->sb)->opt, DIRECT_IO) &&
59 rq->iocb.ki_filp->f_mode & FMODE_CAN_ODIRECT)
60 rq->iocb.ki_flags = IOCB_DIRECT;
61 iov_iter_bvec(&iter, ITER_DEST, rq->bvecs, rq->bio.bi_vcnt,
62 rq->bio.bi_iter.bi_size);
63 ret = vfs_iocb_iter_read(rq->iocb.ki_filp, &rq->iocb, &iter);
64 if (ret != -EIOCBQUEUED)
65 erofs_fileio_ki_complete(&rq->iocb, ret);
66 }
67
erofs_fileio_rq_alloc(struct erofs_map_dev * mdev)68 static struct erofs_fileio_rq *erofs_fileio_rq_alloc(struct erofs_map_dev *mdev)
69 {
70 struct erofs_fileio_rq *rq = kzalloc(sizeof(*rq),
71 GFP_KERNEL | __GFP_NOFAIL);
72
73 bio_init(&rq->bio, NULL, rq->bvecs, ARRAY_SIZE(rq->bvecs), REQ_OP_READ);
74 rq->iocb.ki_filp = mdev->m_dif->file;
75 rq->sb = mdev->m_sb;
76 return rq;
77 }
78
erofs_fileio_bio_alloc(struct erofs_map_dev * mdev)79 struct bio *erofs_fileio_bio_alloc(struct erofs_map_dev *mdev)
80 {
81 return &erofs_fileio_rq_alloc(mdev)->bio;
82 }
83
erofs_fileio_submit_bio(struct bio * bio)84 void erofs_fileio_submit_bio(struct bio *bio)
85 {
86 return erofs_fileio_rq_submit(container_of(bio, struct erofs_fileio_rq,
87 bio));
88 }
89
erofs_fileio_scan_folio(struct erofs_fileio * io,struct folio * folio)90 static int erofs_fileio_scan_folio(struct erofs_fileio *io, struct folio *folio)
91 {
92 struct inode *inode = folio_inode(folio);
93 struct erofs_map_blocks *map = &io->map;
94 unsigned int cur = 0, end = folio_size(folio), len, attached = 0;
95 loff_t pos = folio_pos(folio), ofs;
96 struct iov_iter iter;
97 struct bio_vec bv;
98 int err = 0;
99
100 erofs_onlinefolio_init(folio);
101 while (cur < end) {
102 if (!in_range(pos + cur, map->m_la, map->m_llen)) {
103 map->m_la = pos + cur;
104 map->m_llen = end - cur;
105 err = erofs_map_blocks(inode, map);
106 if (err)
107 break;
108 }
109
110 ofs = folio_pos(folio) + cur - map->m_la;
111 len = min_t(loff_t, map->m_llen - ofs, end - cur);
112 if (map->m_flags & EROFS_MAP_META) {
113 struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
114 void *src;
115
116 src = erofs_read_metabuf(&buf, inode->i_sb,
117 map->m_pa + ofs, EROFS_KMAP);
118 if (IS_ERR(src)) {
119 err = PTR_ERR(src);
120 break;
121 }
122 bvec_set_folio(&bv, folio, len, cur);
123 iov_iter_bvec(&iter, ITER_DEST, &bv, 1, len);
124 if (copy_to_iter(src, len, &iter) != len) {
125 erofs_put_metabuf(&buf);
126 err = -EIO;
127 break;
128 }
129 erofs_put_metabuf(&buf);
130 } else if (!(map->m_flags & EROFS_MAP_MAPPED)) {
131 folio_zero_segment(folio, cur, cur + len);
132 attached = 0;
133 } else {
134 if (io->rq && (map->m_pa + ofs != io->dev.m_pa ||
135 map->m_deviceid != io->dev.m_deviceid)) {
136 io_retry:
137 erofs_fileio_rq_submit(io->rq);
138 io->rq = NULL;
139 }
140
141 if (!io->rq) {
142 io->dev = (struct erofs_map_dev) {
143 .m_pa = io->map.m_pa + ofs,
144 .m_deviceid = io->map.m_deviceid,
145 };
146 err = erofs_map_dev(inode->i_sb, &io->dev);
147 if (err)
148 break;
149 io->rq = erofs_fileio_rq_alloc(&io->dev);
150 io->rq->bio.bi_iter.bi_sector = io->dev.m_pa >> 9;
151 attached = 0;
152 }
153 if (!attached++)
154 erofs_onlinefolio_split(folio);
155 if (!bio_add_folio(&io->rq->bio, folio, len, cur))
156 goto io_retry;
157 io->dev.m_pa += len;
158 }
159 cur += len;
160 }
161 erofs_onlinefolio_end(folio, err);
162 return err;
163 }
164
erofs_fileio_read_folio(struct file * file,struct folio * folio)165 static int erofs_fileio_read_folio(struct file *file, struct folio *folio)
166 {
167 struct erofs_fileio io = {};
168 int err;
169
170 trace_erofs_read_folio(folio, true);
171 err = erofs_fileio_scan_folio(&io, folio);
172 erofs_fileio_rq_submit(io.rq);
173 return err;
174 }
175
erofs_fileio_readahead(struct readahead_control * rac)176 static void erofs_fileio_readahead(struct readahead_control *rac)
177 {
178 struct inode *inode = rac->mapping->host;
179 struct erofs_fileio io = {};
180 struct folio *folio;
181 int err;
182
183 trace_erofs_readpages(inode, readahead_index(rac),
184 readahead_count(rac), true);
185 while ((folio = readahead_folio(rac))) {
186 err = erofs_fileio_scan_folio(&io, folio);
187 if (err && err != -EINTR)
188 erofs_err(inode->i_sb, "readahead error at folio %lu @ nid %llu",
189 folio->index, EROFS_I(inode)->nid);
190 }
191 erofs_fileio_rq_submit(io.rq);
192 }
193
194 const struct address_space_operations erofs_fileio_aops = {
195 .read_folio = erofs_fileio_read_folio,
196 .readahead = erofs_fileio_readahead,
197 };
198