1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0+
2
3 /*
4 * Scatterlist filler helpers for virtio-media.
5 *
6 * Copyright (c) 2023-2024 Google LLC.
7 */
8
9 #include <linux/scatterlist.h>
10 #include <linux/moduleparam.h>
11 #include <media/videobuf2-memops.h>
12
13 #include "scatterlist_filler.h"
14 #include "session.h"
15
16 /**
17 * If set to `true`, then the driver will always copy the data passed to the
18 * host into the shadow buffer (instead of trying to map the source memory into
19 * the SG table directly).
20 */
21 static bool always_use_shadow_buffer = false;
22 module_param(always_use_shadow_buffer, bool, 0660);
23
24 /* Convert a V4L2 IOCTL into the IOCTL code we can give to the host */
25 #define VIRTIO_MEDIA_IOCTL_CODE(IOCTL) ((IOCTL >> _IOC_NRSHIFT) & _IOC_NRMASK)
26
scatterlist_filler_add_sg(struct scatterlist_filler * filler,struct scatterlist * sg)27 int scatterlist_filler_add_sg(struct scatterlist_filler *filler,
28 struct scatterlist *sg)
29 {
30 if (filler->cur_sg >= filler->num_sgs)
31 return -ENOSPC;
32 filler->sgs[filler->cur_sg++] = sg;
33
34 return 0;
35 }
36
scatterlist_filler_add_data(struct scatterlist_filler * filler,void * data,size_t len)37 int scatterlist_filler_add_data(struct scatterlist_filler *filler, void *data,
38 size_t len)
39 {
40 BUG_ON(len == 0);
41
42 if (filler->cur_sg >= filler->num_sgs)
43 return -ENOMEM;
44
45 if (filler->cur_desc >= filler->num_descs)
46 return -ENOSPC;
47 filler->sgs[filler->cur_sg] = &filler->descs[filler->cur_desc];
48
49 if (!always_use_shadow_buffer && virt_addr_valid(data + len)) {
50 /*
51 * If "data" is in the 1:1 physical memory mapping then we can
52 * use a single SG entry and avoid copying.
53 */
54 struct page *page = virt_to_page(data);
55 size_t offset = (((size_t)data) & ~PAGE_MASK);
56 struct scatterlist *next_desc =
57 &filler->descs[filler->cur_desc];
58
59 memset(next_desc, 0, sizeof(*next_desc));
60 sg_set_page(next_desc, page, len, offset);
61 filler->cur_desc++;
62 } else if (!always_use_shadow_buffer && is_vmalloc_addr(data)) {
63 int prev_pfn = -2;
64
65 /*
66 * If "data" has been vmalloc'ed, we need one at most entry per
67 * memory page but can avoid copying.
68 */
69 while (len > 0) {
70 struct page *page = vmalloc_to_page(data);
71 int cur_pfn = page_to_pfn(page);
72 /* All pages but the first will start at offset 0. */
73 unsigned long offset =
74 (((unsigned long)data) & ~PAGE_MASK);
75 size_t len_in_page = min(PAGE_SIZE - offset, len);
76 struct scatterlist *next_desc =
77 &filler->descs[filler->cur_desc];
78
79 if (filler->cur_desc >= filler->num_descs)
80 return -ENOSPC;
81
82 /* Optimize contiguous pages */
83 if (cur_pfn == prev_pfn + 1) {
84 (next_desc - 1)->length += len_in_page;
85 } else {
86 memset(next_desc, 0, sizeof(*next_desc));
87 sg_set_page(next_desc, page, len_in_page,
88 offset);
89 filler->cur_desc++;
90 }
91 data += len_in_page;
92 len -= len_in_page;
93 prev_pfn = cur_pfn;
94 }
95 } else {
96 /*
97 * As a last resort, copy into the shadow buffer and reference
98 * it with a single SG entry.
99 */
100 void *shadow_buffer =
101 filler->shadow_buffer + filler->shadow_buffer_pos;
102 struct page *page = virt_to_page(shadow_buffer);
103 unsigned long offset =
104 (((unsigned long)shadow_buffer) & ~PAGE_MASK);
105 struct scatterlist *next_desc =
106 &filler->descs[filler->cur_desc];
107
108 if (len >
109 filler->shadow_buffer_size - filler->shadow_buffer_pos)
110 return -ENOSPC;
111
112 memcpy(shadow_buffer, data, len);
113 memset(next_desc, 0, sizeof(*next_desc));
114 sg_set_page(next_desc, page, len, offset);
115 filler->cur_desc++;
116 filler->shadow_buffer_pos += len;
117 }
118
119 sg_mark_end(&filler->descs[filler->cur_desc - 1]);
120 filler->cur_sg++;
121
122 return 0;
123 }
124
scatterlist_filler_add_ioctl_cmd(struct scatterlist_filler * filler,struct virtio_media_session * session,u32 ioctl_code)125 int scatterlist_filler_add_ioctl_cmd(struct scatterlist_filler *filler,
126 struct virtio_media_session *session,
127 u32 ioctl_code)
128 {
129 struct virtio_media_cmd_ioctl *cmd_ioctl = &session->cmd.ioctl;
130
131 cmd_ioctl->hdr.cmd = VIRTIO_MEDIA_CMD_IOCTL;
132 cmd_ioctl->session_id = session->id;
133 cmd_ioctl->code = VIRTIO_MEDIA_IOCTL_CODE(ioctl_code);
134
135 return scatterlist_filler_add_data(filler, cmd_ioctl,
136 sizeof(*cmd_ioctl));
137 }
138
scatterlist_filler_add_ioctl_resp(struct scatterlist_filler * filler,struct virtio_media_session * session)139 int scatterlist_filler_add_ioctl_resp(struct scatterlist_filler *filler,
140 struct virtio_media_session *session)
141 {
142 struct virtio_media_resp_ioctl *resp_ioctl = &session->resp.ioctl;
143
144 return scatterlist_filler_add_data(filler, resp_ioctl,
145 sizeof(*resp_ioctl));
146 }
147
scatterlist_filler_retrieve_data(struct virtio_media_session * session,struct scatterlist * sg,void * data,size_t len)148 int scatterlist_filler_retrieve_data(struct virtio_media_session *session,
149 struct scatterlist *sg, void *data,
150 size_t len)
151 {
152 void *shadow_buf = session->shadow_buf;
153 void *kaddr = pfn_to_kaddr(page_to_pfn(sg_page(sg))) + sg->offset;
154
155 BUG_ON(len == 0);
156
157 /*
158 * If our SG entry points inside the shadow buffer, copy the data back to its
159 * origin.
160 */
161 if (kaddr >= shadow_buf &&
162 kaddr < shadow_buf + VIRTIO_SHADOW_BUF_SIZE) {
163 if (kaddr + len >= shadow_buf + VIRTIO_SHADOW_BUF_SIZE)
164 return -EINVAL;
165
166 BUG_ON(sg->length != len);
167
168 memcpy(data, kaddr, len);
169 }
170
171 return 0;
172 }
173
174 /*
175 * num_planes if the number of planes in the original buffer provided by user-space.
176 *
177 * @num_buffer_sgs: number of SGs that were used by this buffer. Useful to know
178 * if we have SG lists for USERPTR buffers that we need to free.
179 */
scatterlist_filler_retrieve_buffer(struct virtio_media_session * session,struct scatterlist ** buffer_sgs,struct v4l2_buffer * b,size_t num_planes)180 int scatterlist_filler_retrieve_buffer(struct virtio_media_session *session,
181 struct scatterlist **buffer_sgs,
182 struct v4l2_buffer *b, size_t num_planes)
183 {
184 struct v4l2_plane *planes = NULL;
185 int i = 0;
186 int ret;
187
188 /* Keep data that will be overwritten but that we need to check later */
189 if (V4L2_TYPE_IS_MULTIPLANAR(b->type)) {
190 planes = b->m.planes;
191 }
192
193 ret = scatterlist_filler_retrieve_data(session, buffer_sgs[i++], b,
194 sizeof(*b));
195 if (ret)
196 return ret;
197
198 if (planes != NULL && num_planes > 0) {
199 b->m.planes = planes;
200 if (b->length > num_planes)
201 return -ENOSPC;
202
203 ret = scatterlist_filler_retrieve_data(
204 session, buffer_sgs[i++], b->m.planes,
205 sizeof(struct v4l2_plane) * num_planes);
206 if (ret)
207 return ret;
208 }
209
210 return 0;
211 }
212
scatterlist_filler_retrieve_ext_ctrls(struct virtio_media_session * session,struct scatterlist ** ctrls_sgs,int num_ctrls_sgs,struct v4l2_ext_controls * ctrls)213 int scatterlist_filler_retrieve_ext_ctrls(struct virtio_media_session *session,
214 struct scatterlist **ctrls_sgs,
215 int num_ctrls_sgs,
216 struct v4l2_ext_controls *ctrls)
217 {
218 struct v4l2_ext_control *controls_backup = ctrls->controls;
219 int i = 0;
220 int ret;
221
222 ret = scatterlist_filler_retrieve_data(session, ctrls_sgs[i++], ctrls,
223 sizeof(*ctrls));
224 if (ret)
225 return ret;
226
227 ctrls->controls = controls_backup;
228
229 if (ctrls->count > 0 && ctrls->controls) {
230 ret = scatterlist_filler_retrieve_data(
231 session, ctrls_sgs[i++], ctrls->controls,
232 sizeof(struct v4l2_ext_control) * ctrls->count);
233 if (ret)
234 return ret;
235 }
236
237 return 0;
238 }
239
240 /**
241 * prepare_userptr_to_host -
242 *
243 * Returns -EFAULT if `userptr` was not a valid user address, which is a case
244 * the driver should consider as "normal" operation. All other failures signal
245 * a problem with the driver.
246 */
prepare_userptr_to_host(struct scatterlist_filler * filler,unsigned long userptr,unsigned long length,struct virtio_media_sg_entry ** sg_list,int * nents)247 static int prepare_userptr_to_host(struct scatterlist_filler *filler,
248 unsigned long userptr, unsigned long length,
249 struct virtio_media_sg_entry **sg_list,
250 int *nents)
251 {
252 struct sg_table sg_table = {};
253 struct frame_vector *framevec;
254 struct scatterlist *sg_iter;
255 struct page **pages;
256 unsigned int pages_count;
257 unsigned int offset = userptr & ~PAGE_MASK;
258 size_t entries_size;
259 int i;
260 int ret;
261
262 framevec = vb2_create_framevec(userptr, length, true);
263 if (IS_ERR(framevec)) {
264 if (PTR_ERR(framevec) != -EFAULT) {
265 printk("error creating frame vector for userptr 0x%lx, length 0x%lx: %ld\n",
266 userptr, length, PTR_ERR(framevec));
267 } else {
268 /* -EINVAL is expected in case of invalid userptr. */
269 framevec = ERR_PTR(-EINVAL);
270 }
271 return PTR_ERR(framevec);
272 }
273
274 pages = frame_vector_pages(framevec);
275 if (IS_ERR(pages)) {
276 printk("error getting vector pages\n");
277 ret = PTR_ERR(pages);
278 goto done;
279 }
280 pages_count = frame_vector_count(framevec);
281 /* TODO turn into a SG list directly here and skip the sg table step */
282 ret = sg_alloc_table_from_pages(&sg_table, pages, pages_count, offset,
283 length, 0);
284 if (ret) {
285 printk("error creating sg table\n");
286 goto done;
287 }
288
289 /* Allocate our actual SG in the shadow buffer. */
290 *nents = sg_nents(sg_table.sgl);
291 entries_size = sizeof(**sg_list) * *nents;
292 if (filler->shadow_buffer_pos + entries_size >
293 filler->shadow_buffer_size) {
294 ret = -ENOMEM;
295 goto free_sg;
296 }
297
298 *sg_list = filler->shadow_buffer + filler->shadow_buffer_pos;
299 filler->shadow_buffer_pos += entries_size;
300
301 for_each_sgtable_sg(&sg_table, sg_iter, i) {
302 struct virtio_media_sg_entry *sg_entry = &(*sg_list)[i];
303 sg_entry->start = sg_phys(sg_iter);
304 sg_entry->len = sg_iter->length;
305 }
306
307 free_sg:
308 sg_free_table(&sg_table);
309
310 done:
311 vb2_destroy_framevec(framevec);
312 return ret;
313 }
314
scatterlist_filler_add_userptr(struct scatterlist_filler * filler,unsigned long userptr,unsigned long length)315 static int scatterlist_filler_add_userptr(struct scatterlist_filler *filler,
316 unsigned long userptr,
317 unsigned long length)
318 {
319 int ret;
320 int nents;
321 struct virtio_media_sg_entry *sg_list;
322
323 ret = prepare_userptr_to_host(filler, userptr, length, &sg_list,
324 &nents);
325 if (ret)
326 return ret;
327
328 ret = scatterlist_filler_add_data(filler, sg_list,
329 sizeof(*sg_list) * nents);
330 if (ret)
331 return ret;
332
333 return 0;
334 }
335
scatterlist_filler_add_buffer(struct scatterlist_filler * filler,struct v4l2_buffer * b)336 int scatterlist_filler_add_buffer(struct scatterlist_filler *filler,
337 struct v4l2_buffer *b)
338 {
339 int i;
340 int ret;
341
342 /* Fixup: plane length must be zero if userptr is NULL */
343 if (!V4L2_TYPE_IS_MULTIPLANAR(b->type) &&
344 b->memory == V4L2_MEMORY_USERPTR && b->m.userptr == 0)
345 b->length = 0;
346
347 /* v4l2_buffer */
348 ret = scatterlist_filler_add_data(filler, b, sizeof(*b));
349 if (ret)
350 return ret;
351
352 if (V4L2_TYPE_IS_MULTIPLANAR(b->type) && b->length > 0) {
353 /* Fixup: plane length must be zero if userptr is NULL */
354 if (b->memory == V4L2_MEMORY_USERPTR) {
355 for (i = 0; i < b->length; i++) {
356 struct v4l2_plane *plane = &b->m.planes[i];
357
358 if (plane->m.userptr == 0)
359 plane->length = 0;
360 }
361 }
362
363 /* Array of v4l2_planes */
364 ret = scatterlist_filler_add_data(filler, b->m.planes,
365 sizeof(struct v4l2_plane) *
366 b->length);
367 if (ret)
368 return ret;
369 }
370
371 return 0;
372 }
373
scatterlist_filler_add_buffer_userptr(struct scatterlist_filler * filler,struct v4l2_buffer * b)374 int scatterlist_filler_add_buffer_userptr(struct scatterlist_filler *filler,
375 struct v4l2_buffer *b)
376 {
377 int i;
378 int ret;
379
380 if (b->memory != V4L2_MEMORY_USERPTR)
381 return 0;
382
383 if (V4L2_TYPE_IS_MULTIPLANAR(b->type)) {
384 for (i = 0; i < b->length; i++) {
385 struct v4l2_plane *plane = &b->m.planes[i];
386 if (b->memory == V4L2_MEMORY_USERPTR &&
387 plane->length > 0) {
388 ret = scatterlist_filler_add_userptr(
389 filler, plane->m.userptr,
390 plane->length);
391 if (ret)
392 return ret;
393 }
394 }
395 } else if (b->length > 0) {
396 ret = scatterlist_filler_add_userptr(filler, b->m.userptr,
397 b->length);
398 if (ret)
399 return ret;
400 }
401
402 return 0;
403 }
404
scatterlist_filler_add_ext_ctrls(struct scatterlist_filler * filler,struct v4l2_ext_controls * ctrls,bool add_userptrs)405 int scatterlist_filler_add_ext_ctrls(struct scatterlist_filler *filler,
406 struct v4l2_ext_controls *ctrls,
407 bool add_userptrs)
408 {
409 int i;
410 int ret;
411
412 /* v4l2_ext_controls */
413 ret = scatterlist_filler_add_data(filler, ctrls, sizeof(*ctrls));
414 if (ret)
415 return ret;
416
417 if (ctrls->count > 0) {
418 /* array of v4l2_controls */
419 ret = scatterlist_filler_add_data(filler, ctrls->controls,
420 sizeof(ctrls->controls[0]) *
421 ctrls->count);
422 if (ret)
423 return ret;
424 }
425
426 if (!add_userptrs)
427 return 0;
428
429 /* Pointers to user memory in individual controls */
430 for (i = 0; i < ctrls->count; i++) {
431 struct v4l2_ext_control *ctrl = &ctrls->controls[i];
432 if (ctrl->size > 0) {
433 ret = scatterlist_filler_add_userptr(
434 filler, (unsigned long)ctrl->ptr, ctrl->size);
435 if (ret)
436 return ret;
437 }
438 }
439
440 return 0;
441 }
442