1 /*
2 * Copyright (c) 2012-2015 Etnaviv Project
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sub license,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the
12 * next paragraph) shall be included in all copies or substantial portions
13 * of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 *
23 * Authors:
24 * Wladimir J. van der Laan <[email protected]>
25 */
26
27 #include "etnaviv_transfer.h"
28 #include "etnaviv_clear_blit.h"
29 #include "etnaviv_context.h"
30 #include "etnaviv_debug.h"
31 #include "etnaviv_etc2.h"
32 #include "etnaviv_screen.h"
33
34 #include "pipe/p_defines.h"
35 #include "util/format/u_formats.h"
36 #include "pipe/p_screen.h"
37 #include "pipe/p_state.h"
38 #include "util/format/u_format.h"
39 #include "util/u_inlines.h"
40 #include "util/u_memory.h"
41 #include "util/u_surface.h"
42 #include "util/u_transfer.h"
43
44 #include "hw/common_3d.xml.h"
45
46 #include "drm-uapi/drm_fourcc.h"
47
48 #define ETNA_PIPE_MAP_DISCARD_LEVEL (PIPE_MAP_DRV_PRV << 0)
49
50 /* Compute offset into a 1D/2D/3D buffer of a certain box.
51 * This box must be aligned to the block width and height of the
52 * underlying format. */
53 static inline size_t
etna_compute_offset(enum pipe_format format,const struct pipe_box * box,size_t stride,size_t layer_stride)54 etna_compute_offset(enum pipe_format format, const struct pipe_box *box,
55 size_t stride, size_t layer_stride)
56 {
57 return box->z * layer_stride +
58 box->y / util_format_get_blockheight(format) * stride +
59 box->x / util_format_get_blockwidth(format) *
60 util_format_get_blocksize(format);
61 }
62
etna_patch_data(void * buffer,const struct pipe_transfer * ptrans)63 static void etna_patch_data(void *buffer, const struct pipe_transfer *ptrans)
64 {
65 struct pipe_resource *prsc = ptrans->resource;
66 struct etna_resource *rsc = etna_resource(prsc);
67 struct etna_resource_level *level = &rsc->levels[ptrans->level];
68
69 if (likely(!etna_etc2_needs_patching(prsc)))
70 return;
71
72 if (level->patched)
73 return;
74
75 /* do have the offsets of blocks to patch? */
76 if (!level->patch_offsets) {
77 level->patch_offsets = CALLOC_STRUCT(util_dynarray);
78
79 etna_etc2_calculate_blocks(buffer, ptrans->stride,
80 ptrans->box.width, ptrans->box.height,
81 prsc->format, level->patch_offsets);
82 }
83
84 etna_etc2_patch(buffer, level->patch_offsets);
85
86 level->patched = true;
87 }
88
etna_unpatch_data(void * buffer,const struct pipe_transfer * ptrans)89 static void etna_unpatch_data(void *buffer, const struct pipe_transfer *ptrans)
90 {
91 struct pipe_resource *prsc = ptrans->resource;
92 struct etna_resource *rsc = etna_resource(prsc);
93 struct etna_resource_level *level = &rsc->levels[ptrans->level];
94
95 if (!level->patched)
96 return;
97
98 etna_etc2_patch(buffer, level->patch_offsets);
99
100 level->patched = false;
101 }
102
103 static void
etna_transfer_unmap(struct pipe_context * pctx,struct pipe_transfer * ptrans)104 etna_transfer_unmap(struct pipe_context *pctx, struct pipe_transfer *ptrans)
105 {
106 struct etna_context *ctx = etna_context(pctx);
107 struct etna_transfer *trans = etna_transfer(ptrans);
108 struct etna_resource *rsc = etna_resource(ptrans->resource);
109 struct etna_resource_level *res_level = &rsc->levels[ptrans->level];
110
111 if (rsc->texture && !etna_resource_newer(rsc, etna_resource(rsc->texture)))
112 rsc = etna_resource(rsc->texture); /* switch to using the texture resource */
113
114 /*
115 * Temporary resources are always pulled into the CPU domain, must push them
116 * back into GPU domain before the RS execs the blit to the base resource.
117 */
118 if (trans->rsc)
119 etna_bo_cpu_fini(etna_resource(trans->rsc)->bo);
120
121 if (ptrans->usage & PIPE_MAP_WRITE) {
122 if (etna_resource_level_needs_flush(res_level)) {
123 if (ptrans->usage & ETNA_PIPE_MAP_DISCARD_LEVEL)
124 etna_resource_level_mark_flushed(res_level);
125 else
126 etna_copy_resource(pctx, &rsc->base, &rsc->base, ptrans->level, ptrans->level);
127 }
128
129 if (trans->rsc) {
130 /* We have a temporary resource due to either tile status or
131 * tiling format. Write back the updated buffer contents.
132 */
133 etna_copy_resource_box(pctx, ptrans->resource, trans->rsc,
134 ptrans->level, 0, &ptrans->box);
135 } else if (trans->staging) {
136 /* map buffer object */
137 if (rsc->layout == ETNA_LAYOUT_TILED) {
138 for (unsigned z = 0; z < ptrans->box.depth; z++) {
139 etna_texture_tile(
140 trans->mapped + (ptrans->box.z + z) * res_level->layer_stride,
141 trans->staging + z * ptrans->layer_stride,
142 ptrans->box.x, ptrans->box.y,
143 res_level->stride, ptrans->box.width, ptrans->box.height,
144 ptrans->stride, util_format_get_blocksize(rsc->base.format));
145 }
146 } else if (rsc->layout == ETNA_LAYOUT_LINEAR) {
147 util_copy_box(trans->mapped, rsc->base.format, res_level->stride,
148 res_level->layer_stride, ptrans->box.x,
149 ptrans->box.y, ptrans->box.z, ptrans->box.width,
150 ptrans->box.height, ptrans->box.depth,
151 trans->staging, ptrans->stride,
152 ptrans->layer_stride, 0, 0, 0 /* src x,y,z */);
153 } else {
154 BUG("unsupported tiling %i", rsc->layout);
155 }
156 }
157
158 if (ptrans->resource->target == PIPE_BUFFER)
159 util_range_add(&rsc->base, &rsc->valid_buffer_range,
160 ptrans->box.x, ptrans->box.x + ptrans->box.width);
161
162 etna_resource_level_ts_mark_invalid(res_level);
163 etna_resource_level_mark_changed(res_level);
164
165 if (rsc->base.bind & PIPE_BIND_SAMPLER_VIEW)
166 ctx->dirty |= ETNA_DIRTY_TEXTURE_CACHES;
167 }
168
169 /* We need to have the patched data ready for the GPU. */
170 etna_patch_data(trans->mapped, ptrans);
171
172 /*
173 * Transfers without a temporary are only pulled into the CPU domain if they
174 * are not mapped unsynchronized. If they are, must push them back into GPU
175 * domain after CPU access is finished.
176 */
177 if (!trans->rsc && !(ptrans->usage & PIPE_MAP_UNSYNCHRONIZED))
178 etna_bo_cpu_fini(rsc->bo);
179
180 FREE(trans->staging);
181 pipe_resource_reference(&trans->rsc, NULL);
182 pipe_resource_reference(&ptrans->resource, NULL);
183 slab_free(&ctx->transfer_pool, trans);
184 }
185
186 static void *
etna_transfer_map(struct pipe_context * pctx,struct pipe_resource * prsc,unsigned level,unsigned usage,const struct pipe_box * box,struct pipe_transfer ** out_transfer)187 etna_transfer_map(struct pipe_context *pctx, struct pipe_resource *prsc,
188 unsigned level,
189 unsigned usage,
190 const struct pipe_box *box,
191 struct pipe_transfer **out_transfer)
192 {
193 struct etna_context *ctx = etna_context(pctx);
194 struct etna_screen *screen = ctx->screen;
195 struct etna_resource *rsc = etna_resource(prsc);
196 struct etna_resource_level *res_level = &rsc->levels[level];
197 struct etna_transfer *trans;
198 struct pipe_transfer *ptrans;
199 enum pipe_format format = prsc->format;
200
201 trans = slab_zalloc(&ctx->transfer_pool);
202 if (!trans)
203 return NULL;
204
205 assert(level <= prsc->last_level);
206
207 /*
208 * Upgrade to UNSYNCHRONIZED if target is PIPE_BUFFER and range is uninitialized.
209 */
210 if ((usage & PIPE_MAP_WRITE) &&
211 (prsc->target == PIPE_BUFFER) &&
212 !util_ranges_intersect(&rsc->valid_buffer_range,
213 box->x,
214 box->x + box->width)) {
215 usage |= PIPE_MAP_UNSYNCHRONIZED;
216 }
217
218 /* Upgrade DISCARD_RANGE to WHOLE_RESOURCE if the whole resource is
219 * being mapped. If we add buffer reallocation to avoid CPU/GPU sync this
220 * check needs to be extended to coherent mappings and shared resources.
221 */
222 if ((usage & PIPE_MAP_DISCARD_RANGE) &&
223 !(usage & PIPE_MAP_UNSYNCHRONIZED) &&
224 !(prsc->flags & PIPE_RESOURCE_FLAG_MAP_PERSISTENT) &&
225 prsc->last_level == 0 &&
226 prsc->width0 == box->width &&
227 prsc->height0 == box->height &&
228 prsc->depth0 == box->depth &&
229 prsc->array_size == 1) {
230 usage |= PIPE_MAP_DISCARD_WHOLE_RESOURCE;
231 }
232
233 if ((usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE) ||
234 ((usage & PIPE_MAP_DISCARD_RANGE) &&
235 util_texrange_covers_whole_level(prsc, level, box->x, box->y, box->z,
236 box->width, box->height, box->depth)))
237 usage |= ETNA_PIPE_MAP_DISCARD_LEVEL;
238
239
240 ptrans = &trans->base;
241 pipe_resource_reference(&ptrans->resource, prsc);
242 ptrans->level = level;
243 ptrans->usage = usage;
244 ptrans->box = *box;
245
246 /* This one is a little tricky: if we have a separate render resource, which
247 * is newer than the base resource we want the transfer to target this one,
248 * to get the most up-to-date content, but only if we don't have a texture
249 * target of the same age, as transfering in/out of the texture target is
250 * generally preferred for the reasons listed below */
251 if (rsc->render && etna_resource_newer(etna_resource(rsc->render), rsc) &&
252 (!rsc->texture || etna_resource_newer(etna_resource(rsc->render),
253 etna_resource(rsc->texture)))) {
254 rsc = etna_resource(rsc->render);
255 }
256
257 if (rsc->texture && !etna_resource_newer(rsc, etna_resource(rsc->texture))) {
258 /* We have a texture resource which is the same age or newer than the
259 * render resource. Use the texture resource, which avoids bouncing
260 * pixels between the two resources, and we can de-tile it in s/w. */
261 rsc = etna_resource(rsc->texture);
262 } else if (etna_resource_level_ts_valid(res_level) ||
263 (rsc->layout != ETNA_LAYOUT_LINEAR &&
264 etna_resource_hw_tileable(screen->specs.use_blt, prsc) &&
265 /* HALIGN 4 resources are incompatible with the resolve engine,
266 * so fall back to using software to detile this resource. */
267 rsc->halign != TEXTURE_HALIGN_FOUR)) {
268
269 /* If the resource level has valid tile status, we copy the transfer
270 * region to a staging resource using the BLT or RS engine, which will
271 * resolve the TS information. We also do this for tiled resources without
272 * TS, but only if the dedicated blit engines can handle them. */
273
274 if (usage & PIPE_MAP_DIRECTLY) {
275 slab_free(&ctx->transfer_pool, trans);
276 BUG("unsupported map flags %#x with tile status/tiled layout", usage);
277 return NULL;
278 }
279
280 struct pipe_resource templ = *prsc;
281 templ.last_level = 0;
282 templ.width0 = res_level->width;
283 templ.height0 = res_level->height;
284 templ.nr_samples = 0;
285 templ.bind = PIPE_BIND_RENDER_TARGET;
286
287 trans->rsc = etna_resource_alloc(pctx->screen, ETNA_LAYOUT_LINEAR,
288 DRM_FORMAT_MOD_LINEAR, &templ);
289 if (!trans->rsc) {
290 slab_free(&ctx->transfer_pool, trans);
291 return NULL;
292 }
293
294 if (!screen->specs.use_blt) {
295 /* Need to align the transfer region to satisfy RS restrictions, as we
296 * really want to hit the RS blit path here.
297 */
298 unsigned w_align, h_align;
299
300 if (rsc->layout & ETNA_LAYOUT_BIT_SUPER) {
301 w_align = 64;
302 h_align = 64 * ctx->screen->specs.pixel_pipes;
303 } else {
304 w_align = ETNA_RS_WIDTH_MASK + 1;
305 h_align = ETNA_RS_HEIGHT_MASK + 1;
306 }
307
308 ptrans->box.width += ptrans->box.x & (w_align - 1);
309 ptrans->box.x = ptrans->box.x & ~(w_align - 1);
310 ptrans->box.width = align(ptrans->box.width, (ETNA_RS_WIDTH_MASK + 1));
311 ptrans->box.height += ptrans->box.y & (h_align - 1);
312 ptrans->box.y = ptrans->box.y & ~(h_align - 1);
313 ptrans->box.height = align(ptrans->box.height, ETNA_RS_HEIGHT_MASK + 1);
314 }
315
316 if ((usage & PIPE_MAP_READ) || !(usage & ETNA_PIPE_MAP_DISCARD_LEVEL))
317 etna_copy_resource_box(pctx, trans->rsc, &rsc->base, 0, level, &ptrans->box);
318
319 /* Switch to using the temporary resource instead */
320 rsc = etna_resource(trans->rsc);
321 res_level = &rsc->levels[0];
322 }
323
324 /* XXX we don't handle PIPE_MAP_FLUSH_EXPLICIT; this flag can be ignored
325 * when mapping in-place,
326 * but when not in place we need to fire off the copy operation in
327 * transfer_flush_region (currently
328 * a no-op) instead of unmap. Need to handle this to support
329 * ARB_map_buffer_range extension at least.
330 */
331
332 /*
333 * Pull resources into the CPU domain. Only skipped for unsynchronized
334 * transfers without a temporary resource.
335 */
336 if (trans->rsc || !(usage & PIPE_MAP_UNSYNCHRONIZED)) {
337 enum etna_resource_status status = etna_resource_status(ctx, rsc);
338 uint32_t prep_flags = 0;
339
340 /*
341 * Always flush if we have the temporary resource and have a copy to this
342 * outstanding. Otherwise infer flush requirement from resource access and
343 * current GPU usage (reads must wait for GPU writes, writes must have
344 * exclusive access to the buffer).
345 */
346 if ((trans->rsc && (status & ETNA_PENDING_WRITE)) ||
347 (!trans->rsc &&
348 (((usage & PIPE_MAP_READ) && (status & ETNA_PENDING_WRITE)) ||
349 ((usage & PIPE_MAP_WRITE) && status)))) {
350 etna_flush(pctx, NULL, 0, true);
351 }
352
353 if (usage & PIPE_MAP_READ)
354 prep_flags |= DRM_ETNA_PREP_READ;
355 if (usage & PIPE_MAP_WRITE)
356 prep_flags |= DRM_ETNA_PREP_WRITE;
357
358 /*
359 * The ETC2 patching operates in-place on the resource, so the resource will
360 * get written even on read-only transfers. This blocks the GPU to sample
361 * from this resource.
362 */
363 if ((usage & PIPE_MAP_READ) && etna_etc2_needs_patching(prsc))
364 prep_flags |= DRM_ETNA_PREP_WRITE;
365
366 if (etna_bo_cpu_prep(rsc->bo, prep_flags))
367 goto fail_prep;
368 }
369
370 /* map buffer object */
371 trans->mapped = etna_bo_map(rsc->bo);
372 if (!trans->mapped)
373 goto fail;
374
375 *out_transfer = ptrans;
376
377 if (rsc->layout == ETNA_LAYOUT_LINEAR) {
378 ptrans->stride = res_level->stride;
379 ptrans->layer_stride = res_level->layer_stride;
380
381 trans->mapped += res_level->offset +
382 etna_compute_offset(prsc->format, box, res_level->stride,
383 res_level->layer_stride);
384
385 /* We need to have the unpatched data ready for the gfx stack. */
386 if (usage & PIPE_MAP_READ)
387 etna_unpatch_data(trans->mapped, ptrans);
388
389 return trans->mapped;
390 } else {
391 unsigned divSizeX = util_format_get_blockwidth(format);
392 unsigned divSizeY = util_format_get_blockheight(format);
393
394 /* No direct mappings of tiled, since we need to manually
395 * tile/untile.
396 */
397 if (usage & PIPE_MAP_DIRECTLY)
398 goto fail;
399
400 trans->mapped += res_level->offset;
401 ptrans->stride = align(box->width, divSizeX) * util_format_get_blocksize(format); /* row stride in bytes */
402 ptrans->layer_stride = align(box->height, divSizeY) * ptrans->stride;
403 size_t size = ptrans->layer_stride * box->depth;
404
405 trans->staging = MALLOC(size);
406 if (!trans->staging)
407 goto fail;
408
409 if (usage & PIPE_MAP_READ) {
410 if (rsc->layout == ETNA_LAYOUT_TILED) {
411 for (unsigned z = 0; z < ptrans->box.depth; z++) {
412 etna_texture_untile(trans->staging + z * ptrans->layer_stride,
413 trans->mapped + (ptrans->box.z + z) * res_level->layer_stride,
414 ptrans->box.x, ptrans->box.y, res_level->stride,
415 ptrans->box.width, ptrans->box.height, ptrans->stride,
416 util_format_get_blocksize(rsc->base.format));
417 }
418 } else if (rsc->layout == ETNA_LAYOUT_LINEAR) {
419 util_copy_box(trans->staging, rsc->base.format, ptrans->stride,
420 ptrans->layer_stride, 0, 0, 0, /* dst x,y,z */
421 ptrans->box.width, ptrans->box.height,
422 ptrans->box.depth, trans->mapped, res_level->stride,
423 res_level->layer_stride, ptrans->box.x,
424 ptrans->box.y, ptrans->box.z);
425 } else {
426 /* TODO supertiling */
427 BUG("unsupported tiling %i for reading", rsc->layout);
428 }
429 }
430
431 return trans->staging;
432 }
433
434 fail:
435 etna_bo_cpu_fini(rsc->bo);
436 fail_prep:
437 etna_transfer_unmap(pctx, ptrans);
438 return NULL;
439 }
440
441 static void
etna_transfer_flush_region(struct pipe_context * pctx,struct pipe_transfer * ptrans,const struct pipe_box * box)442 etna_transfer_flush_region(struct pipe_context *pctx,
443 struct pipe_transfer *ptrans,
444 const struct pipe_box *box)
445 {
446 struct etna_resource *rsc = etna_resource(ptrans->resource);
447
448 if (ptrans->resource->target == PIPE_BUFFER)
449 util_range_add(&rsc->base,
450 &rsc->valid_buffer_range,
451 ptrans->box.x + box->x,
452 ptrans->box.x + box->x + box->width);
453 }
454
455 void
etna_transfer_init(struct pipe_context * pctx)456 etna_transfer_init(struct pipe_context *pctx)
457 {
458 pctx->buffer_map = etna_transfer_map;
459 pctx->texture_map = etna_transfer_map;
460 pctx->transfer_flush_region = etna_transfer_flush_region;
461 pctx->buffer_unmap = etna_transfer_unmap;
462 pctx->texture_unmap = etna_transfer_unmap;
463 pctx->buffer_subdata = u_default_buffer_subdata;
464 pctx->texture_subdata = u_default_texture_subdata;
465 }
466