1 /*
2 * Copyright © 2014-2017 Broadcom
3 * Copyright (C) 2012 Rob Clark <[email protected]>
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22 * IN THE SOFTWARE.
23 */
24
25 #include "pipe/p_defines.h"
26 #include "util/u_memory.h"
27 #include "util/format/u_format.h"
28 #include "util/u_inlines.h"
29 #include "util/u_resource.h"
30 #include "util/u_surface.h"
31 #include "util/u_transfer_helper.h"
32 #include "util/u_upload_mgr.h"
33 #include "util/format/u_format_zs.h"
34 #include "util/u_drm.h"
35
36 #include "drm-uapi/drm_fourcc.h"
37 #include "v3d_screen.h"
38 #include "v3d_context.h"
39 #include "v3d_resource.h"
40 /* The packets used here the same across V3D versions. */
41 #include "broadcom/cle/v3d_packet_v42_pack.h"
42
43 static void
v3d_debug_resource_layout(struct v3d_resource * rsc,const char * caller)44 v3d_debug_resource_layout(struct v3d_resource *rsc, const char *caller)
45 {
46 if (!V3D_DBG(SURFACE))
47 return;
48
49 struct pipe_resource *prsc = &rsc->base;
50
51 if (prsc->target == PIPE_BUFFER) {
52 fprintf(stderr,
53 "rsc %s %p (format %s), %dx%d buffer @0x%08x-0x%08x\n",
54 caller, rsc,
55 util_format_short_name(prsc->format),
56 prsc->width0, prsc->height0,
57 rsc->bo->offset,
58 rsc->bo->offset + rsc->bo->size - 1);
59 return;
60 }
61
62 static const char *const tiling_descriptions[] = {
63 [V3D_TILING_RASTER] = "R",
64 [V3D_TILING_LINEARTILE] = "LT",
65 [V3D_TILING_UBLINEAR_1_COLUMN] = "UB1",
66 [V3D_TILING_UBLINEAR_2_COLUMN] = "UB2",
67 [V3D_TILING_UIF_NO_XOR] = "UIF",
68 [V3D_TILING_UIF_XOR] = "UIF^",
69 };
70
71 for (int i = 0; i <= prsc->last_level; i++) {
72 struct v3d_resource_slice *slice = &rsc->slices[i];
73
74 int level_width = slice->stride / rsc->cpp;
75 int level_height = slice->padded_height;
76 int level_depth =
77 u_minify(util_next_power_of_two(prsc->depth0), i);
78
79 fprintf(stderr,
80 "rsc %s %p (format %s), %dx%d: "
81 "level %d (%s) %dx%dx%d -> %dx%dx%d, stride %d@0x%08x\n",
82 caller, rsc,
83 util_format_short_name(prsc->format),
84 prsc->width0, prsc->height0,
85 i, tiling_descriptions[slice->tiling],
86 u_minify(prsc->width0, i),
87 u_minify(prsc->height0, i),
88 u_minify(prsc->depth0, i),
89 level_width,
90 level_height,
91 level_depth,
92 slice->stride,
93 rsc->bo->offset + slice->offset);
94 }
95 }
96
97 static bool
v3d_resource_bo_alloc(struct v3d_resource * rsc)98 v3d_resource_bo_alloc(struct v3d_resource *rsc)
99 {
100 struct pipe_resource *prsc = &rsc->base;
101 struct pipe_screen *pscreen = prsc->screen;
102 struct v3d_bo *bo;
103
104 /* Buffers may be read using ldunifa, which prefetches the next
105 * 4 bytes after a read. If the buffer's size is exactly a multiple
106 * of a page size and the shader reads the last 4 bytes with ldunifa
107 * the prefetching would read out of bounds and cause an MMU error,
108 * so we allocate extra space to avoid kernel error spamming.
109 */
110 uint32_t size = rsc->size;
111 if (rsc->base.target == PIPE_BUFFER && (size % 4096 == 0))
112 size += 4;
113
114 bo = v3d_bo_alloc(v3d_screen(pscreen), size, "resource");
115 if (bo) {
116 v3d_bo_unreference(&rsc->bo);
117 rsc->bo = bo;
118 rsc->serial_id++;
119 v3d_debug_resource_layout(rsc, "alloc");
120 return true;
121 } else {
122 return false;
123 }
124 }
125
126 static void
v3d_resource_transfer_unmap(struct pipe_context * pctx,struct pipe_transfer * ptrans)127 v3d_resource_transfer_unmap(struct pipe_context *pctx,
128 struct pipe_transfer *ptrans)
129 {
130 struct v3d_context *v3d = v3d_context(pctx);
131 struct v3d_transfer *trans = v3d_transfer(ptrans);
132
133 if (trans->map) {
134 struct v3d_resource *rsc = v3d_resource(ptrans->resource);
135 struct v3d_resource_slice *slice = &rsc->slices[ptrans->level];
136
137 if (ptrans->usage & PIPE_MAP_WRITE) {
138 for (int z = 0; z < ptrans->box.depth; z++) {
139 void *dst = rsc->bo->map +
140 v3d_layer_offset(&rsc->base,
141 ptrans->level,
142 ptrans->box.z + z);
143 v3d_store_tiled_image(dst,
144 slice->stride,
145 (trans->map +
146 ptrans->stride *
147 ptrans->box.height * z),
148 ptrans->stride,
149 slice->tiling, rsc->cpp,
150 slice->padded_height,
151 &ptrans->box);
152 }
153 }
154 free(trans->map);
155 }
156
157 pipe_resource_reference(&ptrans->resource, NULL);
158 slab_free(&v3d->transfer_pool, ptrans);
159 }
160
161 static void
rebind_sampler_views(struct v3d_context * v3d,struct v3d_resource * rsc)162 rebind_sampler_views(struct v3d_context *v3d,
163 struct v3d_resource *rsc)
164 {
165 for (int st = 0; st < PIPE_SHADER_TYPES; st++) {
166 struct v3d_texture_stateobj *tex = v3d->tex + st;
167
168 for (unsigned i = 0; i < tex->num_textures; i++) {
169 struct pipe_sampler_view *psview = tex->textures[i];
170
171 if (psview->texture != &rsc->base)
172 continue;
173
174 struct v3d_sampler_view *sview =
175 v3d_sampler_view(psview);
176
177 if (sview->serial_id == rsc->serial_id)
178 continue;
179
180 struct v3d_device_info *devinfo =
181 &v3d->screen->devinfo;
182
183 v3d_X(devinfo, create_texture_shader_state_bo)(v3d, sview);
184
185 v3d_flag_dirty_sampler_state(v3d, st);
186 }
187 }
188 }
189
190 static void
v3d_map_usage_prep(struct pipe_context * pctx,struct pipe_resource * prsc,unsigned usage)191 v3d_map_usage_prep(struct pipe_context *pctx,
192 struct pipe_resource *prsc,
193 unsigned usage)
194 {
195 struct v3d_context *v3d = v3d_context(pctx);
196 struct v3d_resource *rsc = v3d_resource(prsc);
197
198 if (usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE) {
199 if (v3d_resource_bo_alloc(rsc)) {
200 /* If it might be bound as one of our vertex buffers
201 * or UBOs, make sure we re-emit vertex buffer state
202 * or uniforms.
203 */
204 if (prsc->bind & PIPE_BIND_VERTEX_BUFFER)
205 v3d->dirty |= V3D_DIRTY_VTXBUF;
206 if (prsc->bind & PIPE_BIND_CONSTANT_BUFFER)
207 v3d->dirty |= V3D_DIRTY_CONSTBUF;
208 /* Since we are changing the texture BO we need to
209 * update any bound samplers to point to the new
210 * BO. Notice we can have samplers that are not
211 * currently bound to the state that won't be
212 * updated. These will be fixed when they are bound in
213 * v3d_set_sampler_views.
214 */
215 if (prsc->bind & PIPE_BIND_SAMPLER_VIEW)
216 rebind_sampler_views(v3d, rsc);
217 } else {
218 /* If we failed to reallocate, flush users so that we
219 * don't violate any syncing requirements.
220 */
221 v3d_flush_jobs_reading_resource(v3d, prsc,
222 V3D_FLUSH_DEFAULT,
223 false);
224 }
225 } else if (!(usage & PIPE_MAP_UNSYNCHRONIZED)) {
226 /* If we're writing and the buffer is being used by the CL, we
227 * have to flush the CL first. If we're only reading, we need
228 * to flush if the CL has written our buffer.
229 */
230 if (usage & PIPE_MAP_WRITE) {
231 v3d_flush_jobs_reading_resource(v3d, prsc,
232 V3D_FLUSH_ALWAYS,
233 false);
234 } else {
235 v3d_flush_jobs_writing_resource(v3d, prsc,
236 V3D_FLUSH_ALWAYS,
237 false);
238 }
239 }
240
241 if (usage & PIPE_MAP_WRITE) {
242 rsc->writes++;
243 rsc->graphics_written = true;
244 rsc->initialized_buffers = ~0;
245 }
246 }
247
248 static void *
v3d_resource_transfer_map(struct pipe_context * pctx,struct pipe_resource * prsc,unsigned level,unsigned usage,const struct pipe_box * box,struct pipe_transfer ** pptrans)249 v3d_resource_transfer_map(struct pipe_context *pctx,
250 struct pipe_resource *prsc,
251 unsigned level, unsigned usage,
252 const struct pipe_box *box,
253 struct pipe_transfer **pptrans)
254 {
255 struct v3d_context *v3d = v3d_context(pctx);
256 struct v3d_resource *rsc = v3d_resource(prsc);
257 struct v3d_transfer *trans;
258 struct pipe_transfer *ptrans;
259 enum pipe_format format = prsc->format;
260 char *buf;
261
262 /* MSAA maps should have been handled by u_transfer_helper. */
263 assert(prsc->nr_samples <= 1);
264
265 /* Upgrade DISCARD_RANGE to WHOLE_RESOURCE if the whole resource is
266 * being mapped.
267 */
268 if ((usage & PIPE_MAP_DISCARD_RANGE) &&
269 !(usage & PIPE_MAP_UNSYNCHRONIZED) &&
270 !(prsc->flags & PIPE_RESOURCE_FLAG_MAP_PERSISTENT) &&
271 prsc->last_level == 0 &&
272 prsc->width0 == box->width &&
273 prsc->height0 == box->height &&
274 prsc->depth0 == box->depth &&
275 prsc->array_size == 1 &&
276 rsc->bo->private) {
277 usage |= PIPE_MAP_DISCARD_WHOLE_RESOURCE;
278 }
279
280 v3d_map_usage_prep(pctx, prsc, usage);
281
282 trans = slab_zalloc(&v3d->transfer_pool);
283 if (!trans)
284 return NULL;
285
286 /* XXX: Handle DONTBLOCK, DISCARD_RANGE, PERSISTENT, COHERENT. */
287
288 ptrans = &trans->base;
289
290 pipe_resource_reference(&ptrans->resource, prsc);
291 ptrans->level = level;
292 ptrans->usage = usage;
293 ptrans->box = *box;
294
295 /* Note that the current kernel implementation is synchronous, so no
296 * need to do syncing stuff here yet.
297 */
298
299 if (usage & PIPE_MAP_UNSYNCHRONIZED)
300 buf = v3d_bo_map_unsynchronized(rsc->bo);
301 else
302 buf = v3d_bo_map(rsc->bo);
303 if (!buf) {
304 fprintf(stderr, "Failed to map bo\n");
305 goto fail;
306 }
307
308 *pptrans = ptrans;
309
310 /* Our load/store routines work on entire compressed blocks. */
311 u_box_pixels_to_blocks(&ptrans->box, &ptrans->box, format);
312
313 struct v3d_resource_slice *slice = &rsc->slices[level];
314 if (rsc->tiled) {
315 /* No direct mappings of tiled, since we need to manually
316 * tile/untile.
317 */
318 if (usage & PIPE_MAP_DIRECTLY)
319 return NULL;
320
321 ptrans->stride = ptrans->box.width * rsc->cpp;
322 ptrans->layer_stride = ptrans->stride * ptrans->box.height;
323
324 trans->map = malloc(ptrans->layer_stride * ptrans->box.depth);
325
326 if (usage & PIPE_MAP_READ) {
327 for (int z = 0; z < ptrans->box.depth; z++) {
328 void *src = rsc->bo->map +
329 v3d_layer_offset(&rsc->base,
330 ptrans->level,
331 ptrans->box.z + z);
332 v3d_load_tiled_image((trans->map +
333 ptrans->stride *
334 ptrans->box.height * z),
335 ptrans->stride,
336 src,
337 slice->stride,
338 slice->tiling, rsc->cpp,
339 slice->padded_height,
340 &ptrans->box);
341 }
342 }
343 return trans->map;
344 } else {
345 ptrans->stride = slice->stride;
346 ptrans->layer_stride = rsc->cube_map_stride;
347
348 return buf + slice->offset +
349 ptrans->box.y * ptrans->stride +
350 ptrans->box.x * rsc->cpp +
351 ptrans->box.z * rsc->cube_map_stride;
352 }
353
354
355 fail:
356 v3d_resource_transfer_unmap(pctx, ptrans);
357 return NULL;
358 }
359
360 static void
v3d_texture_subdata(struct pipe_context * pctx,struct pipe_resource * prsc,unsigned level,unsigned usage,const struct pipe_box * box,const void * data,unsigned stride,uintptr_t layer_stride)361 v3d_texture_subdata(struct pipe_context *pctx,
362 struct pipe_resource *prsc,
363 unsigned level,
364 unsigned usage,
365 const struct pipe_box *box,
366 const void *data,
367 unsigned stride,
368 uintptr_t layer_stride)
369 {
370 struct v3d_resource *rsc = v3d_resource(prsc);
371 struct v3d_resource_slice *slice = &rsc->slices[level];
372
373 /* For a direct mapping, we can just take the u_transfer path. */
374 if (!rsc->tiled) {
375 return u_default_texture_subdata(pctx, prsc, level, usage, box,
376 data, stride, layer_stride);
377 }
378
379 /* Otherwise, map and store the texture data directly into the tiled
380 * texture. Note that gallium's texture_subdata may be called with
381 * obvious usage flags missing!
382 */
383 v3d_map_usage_prep(pctx, prsc, usage | (PIPE_MAP_WRITE |
384 PIPE_MAP_DISCARD_RANGE));
385
386 void *buf;
387 if (usage & PIPE_MAP_UNSYNCHRONIZED)
388 buf = v3d_bo_map_unsynchronized(rsc->bo);
389 else
390 buf = v3d_bo_map(rsc->bo);
391
392 for (int i = 0; i < box->depth; i++) {
393 v3d_store_tiled_image(buf +
394 v3d_layer_offset(&rsc->base,
395 level,
396 box->z + i),
397 slice->stride,
398 (void *)data + layer_stride * i,
399 stride,
400 slice->tiling, rsc->cpp, slice->padded_height,
401 box);
402 }
403 }
404
405 static void
v3d_resource_destroy(struct pipe_screen * pscreen,struct pipe_resource * prsc)406 v3d_resource_destroy(struct pipe_screen *pscreen,
407 struct pipe_resource *prsc)
408 {
409 struct v3d_screen *screen = v3d_screen(pscreen);
410 struct v3d_resource *rsc = v3d_resource(prsc);
411
412 if (rsc->scanout)
413 renderonly_scanout_destroy(rsc->scanout, screen->ro);
414
415 v3d_bo_unreference(&rsc->bo);
416 free(rsc);
417 }
418
419 static uint64_t
v3d_resource_modifier(struct v3d_resource * rsc)420 v3d_resource_modifier(struct v3d_resource *rsc)
421 {
422 if (rsc->tiled) {
423 /* A shared tiled buffer should always be allocated as UIF,
424 * not UBLINEAR or LT.
425 */
426 assert(rsc->slices[0].tiling == V3D_TILING_UIF_XOR ||
427 rsc->slices[0].tiling == V3D_TILING_UIF_NO_XOR);
428 return DRM_FORMAT_MOD_BROADCOM_UIF;
429 } else {
430 return DRM_FORMAT_MOD_LINEAR;
431 }
432 }
433
434 static bool
v3d_resource_get_handle(struct pipe_screen * pscreen,struct pipe_context * pctx,struct pipe_resource * prsc,struct winsys_handle * whandle,unsigned usage)435 v3d_resource_get_handle(struct pipe_screen *pscreen,
436 struct pipe_context *pctx,
437 struct pipe_resource *prsc,
438 struct winsys_handle *whandle,
439 unsigned usage)
440 {
441 struct v3d_screen *screen = v3d_screen(pscreen);
442 struct v3d_resource *rsc = v3d_resource(prsc);
443 struct v3d_bo *bo = rsc->bo;
444
445 whandle->stride = rsc->slices[0].stride;
446 whandle->offset = 0;
447 whandle->modifier = v3d_resource_modifier(rsc);
448
449 /* If we're passing some reference to our BO out to some other part of
450 * the system, then we can't do any optimizations about only us being
451 * the ones seeing it (like BO caching).
452 */
453 bo->private = false;
454
455 switch (whandle->type) {
456 case WINSYS_HANDLE_TYPE_SHARED:
457 return v3d_bo_flink(bo, &whandle->handle);
458 case WINSYS_HANDLE_TYPE_KMS:
459 if (screen->ro) {
460 if (renderonly_get_handle(rsc->scanout, whandle)) {
461 whandle->stride = rsc->slices[0].stride;
462 return true;
463 }
464 return false;
465 }
466 whandle->handle = bo->handle;
467 return true;
468 case WINSYS_HANDLE_TYPE_FD:
469 whandle->handle = v3d_bo_get_dmabuf(bo);
470 return whandle->handle != -1;
471 }
472
473 return false;
474 }
475
476 static bool
v3d_resource_get_param(struct pipe_screen * pscreen,struct pipe_context * pctx,struct pipe_resource * prsc,unsigned plane,unsigned layer,unsigned level,enum pipe_resource_param param,unsigned usage,uint64_t * value)477 v3d_resource_get_param(struct pipe_screen *pscreen,
478 struct pipe_context *pctx, struct pipe_resource *prsc,
479 unsigned plane, unsigned layer, unsigned level,
480 enum pipe_resource_param param,
481 unsigned usage, uint64_t *value)
482 {
483 struct v3d_resource *rsc =
484 (struct v3d_resource *)util_resource_at_index(prsc, plane);
485
486 switch (param) {
487 case PIPE_RESOURCE_PARAM_STRIDE:
488 *value = rsc->slices[level].stride;
489 return true;
490 case PIPE_RESOURCE_PARAM_OFFSET:
491 *value = rsc->slices[level].offset;
492 return true;
493 case PIPE_RESOURCE_PARAM_MODIFIER:
494 *value = v3d_resource_modifier(rsc);
495 return true;
496 case PIPE_RESOURCE_PARAM_NPLANES:
497 *value = util_resource_num(prsc);
498 return true;
499 default:
500 return false;
501 }
502 }
503
504 #define PAGE_UB_ROWS (V3D_UIFCFG_PAGE_SIZE / V3D_UIFBLOCK_ROW_SIZE)
505 #define PAGE_UB_ROWS_TIMES_1_5 ((PAGE_UB_ROWS * 3) >> 1)
506 #define PAGE_CACHE_UB_ROWS (V3D_PAGE_CACHE_SIZE / V3D_UIFBLOCK_ROW_SIZE)
507 #define PAGE_CACHE_MINUS_1_5_UB_ROWS (PAGE_CACHE_UB_ROWS - PAGE_UB_ROWS_TIMES_1_5)
508
509 /**
510 * Computes the HW's UIFblock padding for a given height/cpp.
511 *
512 * The goal of the padding is to keep pages of the same color (bank number) at
513 * least half a page away from each other vertically when crossing between
514 * between columns of UIF blocks.
515 */
516 static uint32_t
v3d_get_ub_pad(struct v3d_resource * rsc,uint32_t height)517 v3d_get_ub_pad(struct v3d_resource *rsc, uint32_t height)
518 {
519 uint32_t utile_h = v3d_utile_height(rsc->cpp);
520 uint32_t uif_block_h = utile_h * 2;
521 uint32_t height_ub = height / uif_block_h;
522
523 uint32_t height_offset_in_pc = height_ub % PAGE_CACHE_UB_ROWS;
524
525 /* For the perfectly-aligned-for-UIF-XOR case, don't add any pad. */
526 if (height_offset_in_pc == 0)
527 return 0;
528
529 /* Try padding up to where we're offset by at least half a page. */
530 if (height_offset_in_pc < PAGE_UB_ROWS_TIMES_1_5) {
531 /* If we fit entirely in the page cache, don't pad. */
532 if (height_ub < PAGE_CACHE_UB_ROWS)
533 return 0;
534 else
535 return PAGE_UB_ROWS_TIMES_1_5 - height_offset_in_pc;
536 }
537
538 /* If we're close to being aligned to page cache size, then round up
539 * and rely on XOR.
540 */
541 if (height_offset_in_pc > PAGE_CACHE_MINUS_1_5_UB_ROWS)
542 return PAGE_CACHE_UB_ROWS - height_offset_in_pc;
543
544 /* Otherwise, we're far enough away (top and bottom) to not need any
545 * padding.
546 */
547 return 0;
548 }
549
550 /**
551 * Computes the dimension with required padding for mip levels.
552 *
553 * This padding is required for width and height dimensions when the mip
554 * level is greater than 1, and for the depth dimension when the mip level
555 * is greater than 0. This function expects to be passed a mip level >= 1.
556 *
557 * Note: Hardware documentation seems to suggest that the third argument
558 * should be the utile dimensions, but through testing it was found that
559 * the block dimension should be used instead.
560 */
561 static uint32_t
v3d_get_dimension_mpad(uint32_t dimension,uint32_t level,uint32_t block_dimension)562 v3d_get_dimension_mpad(uint32_t dimension, uint32_t level, uint32_t block_dimension)
563 {
564 assert(level >= 1);
565 uint32_t pot_dim = u_minify(dimension, 1);
566 pot_dim = util_next_power_of_two(DIV_ROUND_UP(pot_dim, block_dimension));
567 uint32_t padded_dim = block_dimension * pot_dim;
568 return u_minify(padded_dim, level - 1);
569 }
570
571 static void
v3d_setup_slices(struct v3d_resource * rsc,uint32_t winsys_stride,bool uif_top)572 v3d_setup_slices(struct v3d_resource *rsc, uint32_t winsys_stride,
573 bool uif_top)
574 {
575 struct pipe_resource *prsc = &rsc->base;
576 uint32_t width = prsc->width0;
577 uint32_t height = prsc->height0;
578 uint32_t depth = prsc->depth0;
579 uint32_t offset = 0;
580 uint32_t utile_w = v3d_utile_width(rsc->cpp);
581 uint32_t utile_h = v3d_utile_height(rsc->cpp);
582 uint32_t uif_block_w = utile_w * 2;
583 uint32_t uif_block_h = utile_h * 2;
584 uint32_t block_width = util_format_get_blockwidth(prsc->format);
585 uint32_t block_height = util_format_get_blockheight(prsc->format);
586
587 /* Note that power-of-two padding is based on level 1. These are not
588 * equivalent to just util_next_power_of_two(dimension), because at a
589 * level 0 dimension of 9, the level 1 power-of-two padded value is 4,
590 * not 8. Additionally the pot padding is based on the block size.
591 */
592 uint32_t pot_width = 2 * v3d_get_dimension_mpad(width,
593 1,
594 block_width);
595 uint32_t pot_height = 2 * v3d_get_dimension_mpad(height,
596 1,
597 block_height);
598 uint32_t pot_depth = 2 * v3d_get_dimension_mpad(depth,
599 1,
600 1);
601 bool msaa = prsc->nr_samples > 1;
602
603 /* MSAA textures/renderbuffers are always laid out as single-level
604 * UIF.
605 */
606 uif_top |= msaa;
607
608 /* Check some easy mistakes to make in a resource_create() call that
609 * will break our setup.
610 */
611 assert(prsc->array_size != 0);
612 assert(prsc->depth0 != 0);
613
614 for (int i = prsc->last_level; i >= 0; i--) {
615 struct v3d_resource_slice *slice = &rsc->slices[i];
616
617 uint32_t level_width, level_height, level_depth;
618 if (i < 2) {
619 level_width = u_minify(width, i);
620 level_height = u_minify(height, i);
621 } else {
622 level_width = u_minify(pot_width, i);
623 level_height = u_minify(pot_height, i);
624 }
625 if (i < 1)
626 level_depth = u_minify(depth, i);
627 else
628 level_depth = u_minify(pot_depth, i);
629
630 if (msaa) {
631 level_width *= 2;
632 level_height *= 2;
633 }
634
635 level_width = DIV_ROUND_UP(level_width, block_width);
636 level_height = DIV_ROUND_UP(level_height, block_height);
637
638 if (!rsc->tiled) {
639 slice->tiling = V3D_TILING_RASTER;
640 if (prsc->target == PIPE_TEXTURE_1D ||
641 prsc->target == PIPE_TEXTURE_1D_ARRAY)
642 level_width = align(level_width, 64 / rsc->cpp);
643 } else {
644 if ((i != 0 || !uif_top) &&
645 (level_width <= utile_w ||
646 level_height <= utile_h)) {
647 slice->tiling = V3D_TILING_LINEARTILE;
648 level_width = align(level_width, utile_w);
649 level_height = align(level_height, utile_h);
650 } else if ((i != 0 || !uif_top) &&
651 level_width <= uif_block_w) {
652 slice->tiling = V3D_TILING_UBLINEAR_1_COLUMN;
653 level_width = align(level_width, uif_block_w);
654 level_height = align(level_height, uif_block_h);
655 } else if ((i != 0 || !uif_top) &&
656 level_width <= 2 * uif_block_w) {
657 slice->tiling = V3D_TILING_UBLINEAR_2_COLUMN;
658 level_width = align(level_width, 2 * uif_block_w);
659 level_height = align(level_height, uif_block_h);
660 } else {
661 /* We align the width to a 4-block column of
662 * UIF blocks, but we only align height to UIF
663 * blocks.
664 */
665 level_width = align(level_width,
666 4 * uif_block_w);
667 level_height = align(level_height,
668 uif_block_h);
669
670 slice->ub_pad = v3d_get_ub_pad(rsc,
671 level_height);
672 level_height += slice->ub_pad * uif_block_h;
673
674 /* If the padding set us to to be aligned to
675 * the page cache size, then the HW will use
676 * the XOR bit on odd columns to get us
677 * perfectly misaligned
678 */
679 if ((level_height / uif_block_h) %
680 (V3D_PAGE_CACHE_SIZE /
681 V3D_UIFBLOCK_ROW_SIZE) == 0) {
682 slice->tiling = V3D_TILING_UIF_XOR;
683 } else {
684 slice->tiling = V3D_TILING_UIF_NO_XOR;
685 }
686 }
687 }
688
689 slice->offset = offset;
690 if (winsys_stride)
691 slice->stride = winsys_stride;
692 else
693 slice->stride = level_width * rsc->cpp;
694 slice->padded_height = level_height;
695 slice->size = level_height * slice->stride;
696
697 uint32_t slice_total_size = slice->size * level_depth;
698
699 /* The HW aligns level 1's base to a page if any of level 1 or
700 * below could be UIF XOR. The lower levels then inherit the
701 * alignment for as long as necessary, thanks to being power of
702 * two aligned.
703 */
704 if (i == 1 &&
705 level_width > 4 * uif_block_w &&
706 level_height > PAGE_CACHE_MINUS_1_5_UB_ROWS * uif_block_h) {
707 slice_total_size = align(slice_total_size,
708 V3D_UIFCFG_PAGE_SIZE);
709 }
710
711 offset += slice_total_size;
712
713 }
714 rsc->size = offset;
715
716 /* UIF/UBLINEAR levels need to be aligned to UIF-blocks, and LT only
717 * needs to be aligned to utile boundaries. Since tiles are laid out
718 * from small to big in memory, we need to align the later UIF slices
719 * to UIF blocks, if they were preceded by non-UIF-block-aligned LT
720 * slices.
721 *
722 * We additionally align to 4k, which improves UIF XOR performance.
723 */
724 uint32_t page_align_offset = (align(rsc->slices[0].offset, 4096) -
725 rsc->slices[0].offset);
726 if (page_align_offset) {
727 rsc->size += page_align_offset;
728 for (int i = 0; i <= prsc->last_level; i++)
729 rsc->slices[i].offset += page_align_offset;
730 }
731
732 /* Arrays and cube textures have a stride which is the distance from
733 * one full mipmap tree to the next (64b aligned). For 3D textures,
734 * we need to program the stride between slices of miplevel 0.
735 */
736 if (prsc->target != PIPE_TEXTURE_3D) {
737 rsc->cube_map_stride = align(rsc->slices[0].offset +
738 rsc->slices[0].size, 64);
739 rsc->size += rsc->cube_map_stride * (prsc->array_size - 1);
740 } else {
741 rsc->cube_map_stride = rsc->slices[0].size;
742 }
743 }
744
745 uint32_t
v3d_layer_offset(struct pipe_resource * prsc,uint32_t level,uint32_t layer)746 v3d_layer_offset(struct pipe_resource *prsc, uint32_t level, uint32_t layer)
747 {
748 struct v3d_resource *rsc = v3d_resource(prsc);
749 struct v3d_resource_slice *slice = &rsc->slices[level];
750
751 if (prsc->target == PIPE_TEXTURE_3D)
752 return slice->offset + layer * slice->size;
753 else
754 return slice->offset + layer * rsc->cube_map_stride;
755 }
756
757 static struct v3d_resource *
v3d_resource_setup(struct pipe_screen * pscreen,const struct pipe_resource * tmpl)758 v3d_resource_setup(struct pipe_screen *pscreen,
759 const struct pipe_resource *tmpl)
760 {
761 struct v3d_resource *rsc = CALLOC_STRUCT(v3d_resource);
762
763 if (!rsc)
764 return NULL;
765 struct pipe_resource *prsc = &rsc->base;
766
767 *prsc = *tmpl;
768
769 pipe_reference_init(&prsc->reference, 1);
770 prsc->screen = pscreen;
771
772 rsc->cpp = util_format_get_blocksize(prsc->format);
773 rsc->serial_id++;
774
775 assert(rsc->cpp);
776
777 return rsc;
778 }
779
780 static struct pipe_resource *
v3d_resource_create_with_modifiers(struct pipe_screen * pscreen,const struct pipe_resource * tmpl,const uint64_t * modifiers,int count)781 v3d_resource_create_with_modifiers(struct pipe_screen *pscreen,
782 const struct pipe_resource *tmpl,
783 const uint64_t *modifiers,
784 int count)
785 {
786 struct v3d_screen *screen = v3d_screen(pscreen);
787
788 bool linear_ok = drm_find_modifier(DRM_FORMAT_MOD_LINEAR, modifiers, count);
789 struct v3d_resource *rsc = v3d_resource_setup(pscreen, tmpl);
790 struct pipe_resource *prsc = &rsc->base;
791 /* Use a tiled layout if we can, for better 3D performance. */
792 bool should_tile = true;
793
794 assert(tmpl->target != PIPE_BUFFER ||
795 (tmpl->format == PIPE_FORMAT_NONE ||
796 util_format_get_blocksize(tmpl->format) == 1));
797
798 /* VBOs/PBOs/Texture Buffer Objects are untiled (and 1 height). */
799 if (tmpl->target == PIPE_BUFFER)
800 should_tile = false;
801
802 /* Cursors are always linear, and the user can request linear as well.
803 */
804 if (tmpl->bind & (PIPE_BIND_LINEAR | PIPE_BIND_CURSOR))
805 should_tile = false;
806
807 /* 1D and 1D_ARRAY textures are always raster-order. */
808 if (tmpl->target == PIPE_TEXTURE_1D ||
809 tmpl->target == PIPE_TEXTURE_1D_ARRAY)
810 should_tile = false;
811
812 /* Scanout BOs for simulator need to be linear for interaction with
813 * i965.
814 */
815 #if USE_V3D_SIMULATOR
816 if (tmpl->bind & PIPE_BIND_SHARED)
817 should_tile = false;
818 #endif
819
820 /* If using the old-school SCANOUT flag, we don't know what the screen
821 * might support other than linear. Just force linear.
822 */
823 if (tmpl->bind & PIPE_BIND_SCANOUT)
824 should_tile = false;
825
826 /* No user-specified modifier; determine our own. */
827 if (count == 1 && modifiers[0] == DRM_FORMAT_MOD_INVALID) {
828 linear_ok = true;
829 rsc->tiled = should_tile;
830 } else if (should_tile &&
831 drm_find_modifier(DRM_FORMAT_MOD_BROADCOM_UIF,
832 modifiers, count)) {
833 rsc->tiled = true;
834 } else if (linear_ok) {
835 rsc->tiled = false;
836 } else {
837 fprintf(stderr, "Unsupported modifier requested\n");
838 goto fail;
839 }
840
841 rsc->internal_format = prsc->format;
842
843 v3d_setup_slices(rsc, 0, tmpl->bind & PIPE_BIND_SHARED);
844
845 if (screen->ro && (tmpl->bind & PIPE_BIND_SCANOUT)) {
846 assert(!rsc->tiled);
847 struct winsys_handle handle;
848 struct pipe_resource scanout_tmpl = {
849 .target = prsc->target,
850 .format = PIPE_FORMAT_RGBA8888_UNORM,
851 .width0 = 1024, /* one page */
852 .height0 = align(rsc->size, 4096) / 4096,
853 .depth0 = 1,
854 .array_size = 1,
855 };
856
857 rsc->scanout =
858 renderonly_scanout_for_resource(&scanout_tmpl,
859 screen->ro,
860 &handle);
861
862 if (!rsc->scanout) {
863 fprintf(stderr, "Failed to create scanout resource\n");
864 goto fail;
865 }
866 assert(handle.type == WINSYS_HANDLE_TYPE_FD);
867 rsc->bo = v3d_bo_open_dmabuf(screen, handle.handle);
868 close(handle.handle);
869
870 if (!rsc->bo)
871 goto fail;
872
873 v3d_debug_resource_layout(rsc, "renderonly");
874
875 return prsc;
876 } else {
877 if (!v3d_resource_bo_alloc(rsc))
878 goto fail;
879 }
880
881 return prsc;
882 fail:
883 v3d_resource_destroy(pscreen, prsc);
884 return NULL;
885 }
886
887 struct pipe_resource *
v3d_resource_create(struct pipe_screen * pscreen,const struct pipe_resource * tmpl)888 v3d_resource_create(struct pipe_screen *pscreen,
889 const struct pipe_resource *tmpl)
890 {
891 const uint64_t mod = DRM_FORMAT_MOD_INVALID;
892 return v3d_resource_create_with_modifiers(pscreen, tmpl, &mod, 1);
893 }
894
895 static struct pipe_resource *
v3d_resource_from_handle(struct pipe_screen * pscreen,const struct pipe_resource * tmpl,struct winsys_handle * whandle,unsigned usage)896 v3d_resource_from_handle(struct pipe_screen *pscreen,
897 const struct pipe_resource *tmpl,
898 struct winsys_handle *whandle,
899 unsigned usage)
900 {
901 struct v3d_screen *screen = v3d_screen(pscreen);
902 struct v3d_resource *rsc = v3d_resource_setup(pscreen, tmpl);
903 struct pipe_resource *prsc = &rsc->base;
904 struct v3d_resource_slice *slice = &rsc->slices[0];
905
906 if (!rsc)
907 return NULL;
908
909 switch (whandle->modifier) {
910 case DRM_FORMAT_MOD_LINEAR:
911 rsc->tiled = false;
912 break;
913 case DRM_FORMAT_MOD_BROADCOM_UIF:
914 rsc->tiled = true;
915 break;
916 case DRM_FORMAT_MOD_INVALID:
917 rsc->tiled = false;
918 break;
919 default:
920 switch(fourcc_mod_broadcom_mod(whandle->modifier)) {
921 case DRM_FORMAT_MOD_BROADCOM_SAND128:
922 rsc->tiled = false;
923 rsc->sand_col128_stride =
924 fourcc_mod_broadcom_param(whandle->modifier);
925 break;
926 default:
927 fprintf(stderr,
928 "Attempt to import unsupported modifier 0x%llx\n",
929 (long long)whandle->modifier);
930 goto fail;
931 }
932 }
933
934 switch (whandle->type) {
935 case WINSYS_HANDLE_TYPE_SHARED:
936 rsc->bo = v3d_bo_open_name(screen, whandle->handle);
937 break;
938 case WINSYS_HANDLE_TYPE_FD:
939 rsc->bo = v3d_bo_open_dmabuf(screen, whandle->handle);
940 break;
941 default:
942 fprintf(stderr,
943 "Attempt to import unsupported handle type %d\n",
944 whandle->type);
945 goto fail;
946 }
947
948 if (!rsc->bo)
949 goto fail;
950
951 rsc->internal_format = prsc->format;
952
953 v3d_setup_slices(rsc, whandle->stride, true);
954 v3d_debug_resource_layout(rsc, "import");
955
956 if (whandle->offset != 0) {
957 if (rsc->tiled) {
958 fprintf(stderr,
959 "Attempt to import unsupported winsys offset %u\n",
960 whandle->offset);
961 goto fail;
962 }
963 rsc->slices[0].offset += whandle->offset;
964
965 if (rsc->slices[0].offset + rsc->slices[0].size >
966 rsc->bo->size) {
967 fprintf(stderr, "Attempt to import "
968 "with overflowing offset (%d + %d > %d)\n",
969 whandle->offset,
970 rsc->slices[0].size,
971 rsc->bo->size);
972 goto fail;
973 }
974 }
975
976 if (screen->ro) {
977 /* Make sure that renderonly has a handle to our buffer in the
978 * display's fd, so that a later renderonly_get_handle()
979 * returns correct handles or GEM names.
980 */
981 rsc->scanout =
982 renderonly_create_gpu_import_for_resource(prsc,
983 screen->ro,
984 NULL);
985 }
986
987 if (rsc->tiled && whandle->stride != slice->stride) {
988 static bool warned = false;
989 if (!warned) {
990 warned = true;
991 fprintf(stderr,
992 "Attempting to import %dx%d %s with "
993 "unsupported stride %d instead of %d\n",
994 prsc->width0, prsc->height0,
995 util_format_short_name(prsc->format),
996 whandle->stride,
997 slice->stride);
998 }
999 goto fail;
1000 } else if (!rsc->tiled) {
1001 slice->stride = whandle->stride;
1002 }
1003
1004 /* Prevent implicit clearing of the imported buffer contents. */
1005 rsc->writes = 1;
1006
1007 return prsc;
1008
1009 fail:
1010 v3d_resource_destroy(pscreen, prsc);
1011 return NULL;
1012 }
1013
1014 void
v3d_update_shadow_texture(struct pipe_context * pctx,struct pipe_sampler_view * pview)1015 v3d_update_shadow_texture(struct pipe_context *pctx,
1016 struct pipe_sampler_view *pview)
1017 {
1018 struct v3d_context *v3d = v3d_context(pctx);
1019 struct v3d_sampler_view *view = v3d_sampler_view(pview);
1020 struct v3d_resource *shadow = v3d_resource(view->texture);
1021 struct v3d_resource *orig = v3d_resource(pview->texture);
1022
1023 assert(view->texture != pview->texture);
1024
1025 if (shadow->writes == orig->writes && orig->bo->private)
1026 return;
1027
1028 perf_debug("Updating %dx%d@%d shadow for linear texture\n",
1029 orig->base.width0, orig->base.height0,
1030 pview->u.tex.first_level);
1031
1032 for (int i = 0; i <= shadow->base.last_level; i++) {
1033 unsigned width = u_minify(shadow->base.width0, i);
1034 unsigned height = u_minify(shadow->base.height0, i);
1035 struct pipe_blit_info info = {
1036 .dst = {
1037 .resource = &shadow->base,
1038 .level = i,
1039 .box = {
1040 .x = 0,
1041 .y = 0,
1042 .z = 0,
1043 .width = width,
1044 .height = height,
1045 .depth = 1,
1046 },
1047 .format = shadow->base.format,
1048 },
1049 .src = {
1050 .resource = &orig->base,
1051 .level = pview->u.tex.first_level + i,
1052 .box = {
1053 .x = 0,
1054 .y = 0,
1055 .z = 0,
1056 .width = width,
1057 .height = height,
1058 .depth = 1,
1059 },
1060 .format = orig->base.format,
1061 },
1062 .mask = util_format_get_mask(orig->base.format),
1063 };
1064 pctx->blit(pctx, &info);
1065 }
1066
1067 shadow->writes = orig->writes;
1068 }
1069
1070 static struct pipe_surface *
v3d_create_surface(struct pipe_context * pctx,struct pipe_resource * ptex,const struct pipe_surface * surf_tmpl)1071 v3d_create_surface(struct pipe_context *pctx,
1072 struct pipe_resource *ptex,
1073 const struct pipe_surface *surf_tmpl)
1074 {
1075 struct v3d_context *v3d = v3d_context(pctx);
1076 struct v3d_screen *screen = v3d->screen;
1077 struct v3d_device_info *devinfo = &screen->devinfo;
1078 struct v3d_surface *surface = CALLOC_STRUCT(v3d_surface);
1079 struct v3d_resource *rsc = v3d_resource(ptex);
1080
1081 if (!surface)
1082 return NULL;
1083
1084 struct pipe_surface *psurf = &surface->base;
1085 unsigned level = surf_tmpl->u.tex.level;
1086 struct v3d_resource_slice *slice = &rsc->slices[level];
1087
1088 pipe_reference_init(&psurf->reference, 1);
1089 pipe_resource_reference(&psurf->texture, ptex);
1090
1091 psurf->context = pctx;
1092 psurf->format = surf_tmpl->format;
1093 psurf->width = u_minify(ptex->width0, level);
1094 psurf->height = u_minify(ptex->height0, level);
1095 psurf->u.tex.level = level;
1096 psurf->u.tex.first_layer = surf_tmpl->u.tex.first_layer;
1097 psurf->u.tex.last_layer = surf_tmpl->u.tex.last_layer;
1098
1099 surface->offset = v3d_layer_offset(ptex, level,
1100 psurf->u.tex.first_layer);
1101 surface->tiling = slice->tiling;
1102
1103 surface->format = v3d_get_rt_format(devinfo, psurf->format);
1104
1105 const struct util_format_description *desc =
1106 util_format_description(psurf->format);
1107
1108 surface->swap_rb = (desc->swizzle[0] == PIPE_SWIZZLE_Z &&
1109 psurf->format != PIPE_FORMAT_B5G6R5_UNORM);
1110
1111 if (util_format_is_depth_or_stencil(psurf->format)) {
1112 switch (psurf->format) {
1113 case PIPE_FORMAT_Z16_UNORM:
1114 surface->internal_type = V3D_INTERNAL_TYPE_DEPTH_16;
1115 break;
1116 case PIPE_FORMAT_Z32_FLOAT:
1117 case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
1118 surface->internal_type = V3D_INTERNAL_TYPE_DEPTH_32F;
1119 break;
1120 default:
1121 surface->internal_type = V3D_INTERNAL_TYPE_DEPTH_24;
1122 }
1123 } else {
1124 uint32_t bpp, type;
1125 v3d_X(devinfo, get_internal_type_bpp_for_output_format)
1126 (surface->format, &type, &bpp);
1127 surface->internal_type = type;
1128 surface->internal_bpp = bpp;
1129 }
1130
1131 if (surface->tiling == V3D_TILING_UIF_NO_XOR ||
1132 surface->tiling == V3D_TILING_UIF_XOR) {
1133 surface->padded_height_of_output_image_in_uif_blocks =
1134 (slice->padded_height /
1135 (2 * v3d_utile_height(rsc->cpp)));
1136 }
1137
1138 if (rsc->separate_stencil) {
1139 surface->separate_stencil =
1140 v3d_create_surface(pctx, &rsc->separate_stencil->base,
1141 surf_tmpl);
1142 }
1143
1144 return &surface->base;
1145 }
1146
1147 static void
v3d_surface_destroy(struct pipe_context * pctx,struct pipe_surface * psurf)1148 v3d_surface_destroy(struct pipe_context *pctx, struct pipe_surface *psurf)
1149 {
1150 struct v3d_surface *surf = v3d_surface(psurf);
1151
1152 if (surf->separate_stencil)
1153 pipe_surface_reference(&surf->separate_stencil, NULL);
1154
1155 pipe_resource_reference(&psurf->texture, NULL);
1156 FREE(psurf);
1157 }
1158
1159 static void
v3d_flush_resource(struct pipe_context * pctx,struct pipe_resource * resource)1160 v3d_flush_resource(struct pipe_context *pctx, struct pipe_resource *resource)
1161 {
1162 /* All calls to flush_resource are followed by a flush of the context,
1163 * so there's nothing to do.
1164 */
1165 }
1166
1167 static enum pipe_format
v3d_resource_get_internal_format(struct pipe_resource * prsc)1168 v3d_resource_get_internal_format(struct pipe_resource *prsc)
1169 {
1170 return v3d_resource(prsc)->internal_format;
1171 }
1172
1173 static void
v3d_resource_set_stencil(struct pipe_resource * prsc,struct pipe_resource * stencil)1174 v3d_resource_set_stencil(struct pipe_resource *prsc,
1175 struct pipe_resource *stencil)
1176 {
1177 v3d_resource(prsc)->separate_stencil = v3d_resource(stencil);
1178 }
1179
1180 static struct pipe_resource *
v3d_resource_get_stencil(struct pipe_resource * prsc)1181 v3d_resource_get_stencil(struct pipe_resource *prsc)
1182 {
1183 struct v3d_resource *rsc = v3d_resource(prsc);
1184
1185 return rsc->separate_stencil ? &rsc->separate_stencil->base : NULL;
1186 }
1187
1188 static const struct u_transfer_vtbl transfer_vtbl = {
1189 .resource_create = v3d_resource_create,
1190 .resource_destroy = v3d_resource_destroy,
1191 .transfer_map = v3d_resource_transfer_map,
1192 .transfer_unmap = v3d_resource_transfer_unmap,
1193 .transfer_flush_region = u_default_transfer_flush_region,
1194 .get_internal_format = v3d_resource_get_internal_format,
1195 .set_stencil = v3d_resource_set_stencil,
1196 .get_stencil = v3d_resource_get_stencil,
1197 };
1198
1199 void
v3d_resource_screen_init(struct pipe_screen * pscreen)1200 v3d_resource_screen_init(struct pipe_screen *pscreen)
1201 {
1202 pscreen->resource_create_with_modifiers =
1203 v3d_resource_create_with_modifiers;
1204 pscreen->resource_create = u_transfer_helper_resource_create;
1205 pscreen->resource_from_handle = v3d_resource_from_handle;
1206 pscreen->resource_get_handle = v3d_resource_get_handle;
1207 pscreen->resource_get_param = v3d_resource_get_param;
1208 pscreen->resource_destroy = u_transfer_helper_resource_destroy;
1209 pscreen->transfer_helper = u_transfer_helper_create(&transfer_vtbl,
1210 U_TRANSFER_HELPER_SEPARATE_Z32S8 |
1211 U_TRANSFER_HELPER_MSAA_MAP);
1212 }
1213
1214 void
v3d_resource_context_init(struct pipe_context * pctx)1215 v3d_resource_context_init(struct pipe_context *pctx)
1216 {
1217 pctx->buffer_map = u_transfer_helper_transfer_map;
1218 pctx->texture_map = u_transfer_helper_transfer_map;
1219 pctx->transfer_flush_region = u_transfer_helper_transfer_flush_region;
1220 pctx->buffer_unmap = u_transfer_helper_transfer_unmap;
1221 pctx->texture_unmap = u_transfer_helper_transfer_unmap;
1222 pctx->buffer_subdata = u_default_buffer_subdata;
1223 pctx->texture_subdata = v3d_texture_subdata;
1224 pctx->create_surface = v3d_create_surface;
1225 pctx->surface_destroy = v3d_surface_destroy;
1226 pctx->resource_copy_region = util_resource_copy_region;
1227 pctx->blit = v3d_blit;
1228 pctx->generate_mipmap = v3d_generate_mipmap;
1229 pctx->flush_resource = v3d_flush_resource;
1230 }
1231