1 /*
2 * Copyright (c) 2012-2017 Etnaviv Project
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sub license,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the
12 * next paragraph) shall be included in all copies or substantial portions
13 * of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 *
23 * Authors:
24 * Wladimir J. van der Laan <[email protected]>
25 */
26
27 #include "etnaviv_rs.h"
28
29 #include "etnaviv_clear_blit.h"
30 #include "etnaviv_context.h"
31 #include "etnaviv_emit.h"
32 #include "etnaviv_format.h"
33 #include "etnaviv_resource.h"
34 #include "etnaviv_screen.h"
35 #include "etnaviv_surface.h"
36 #include "etnaviv_tiling.h"
37 #include "etnaviv_translate.h"
38 #include "etnaviv_util.h"
39
40 #include "pipe/p_defines.h"
41 #include "pipe/p_state.h"
42 #include "util/compiler.h"
43 #include "util/u_blitter.h"
44 #include "util/u_inlines.h"
45 #include "util/u_memory.h"
46 #include "util/u_surface.h"
47
48 #include "hw/state.xml.h"
49 #include "hw/state_3d.xml.h"
50
51 #include <assert.h>
52
53 /* return a RS "compatible" format for use when copying */
54 static uint32_t
etna_compatible_rs_format(enum pipe_format fmt)55 etna_compatible_rs_format(enum pipe_format fmt)
56 {
57 /* YUYV and UYVY are blocksize 4, but 2 bytes per pixel */
58 if (fmt == PIPE_FORMAT_YUYV || fmt == PIPE_FORMAT_UYVY)
59 return RS_FORMAT_A4R4G4B4;
60
61 switch (util_format_get_blocksize(fmt)) {
62 case 2: return RS_FORMAT_A4R4G4B4;
63 case 4: return RS_FORMAT_A8R8G8B8;
64 default: return ETNA_NO_MATCH;
65 }
66 }
67
68 void
etna_compile_rs_state(struct etna_context * ctx,struct compiled_rs_state * cs,const struct rs_state * rs)69 etna_compile_rs_state(struct etna_context *ctx, struct compiled_rs_state *cs,
70 const struct rs_state *rs)
71 {
72 struct etna_screen *screen = ctx->screen;
73
74 memset(cs, 0, sizeof(*cs));
75
76 /* TILED and SUPERTILED layout have their strides multiplied with 4 in RS */
77 unsigned source_stride_shift = COND(rs->source_tiling != ETNA_LAYOUT_LINEAR, 2);
78 unsigned dest_stride_shift = COND(rs->dest_tiling != ETNA_LAYOUT_LINEAR, 2);
79
80 bool src_tiled = rs->source_tiling & ETNA_LAYOUT_BIT_TILE;
81 bool dst_tiled = rs->dest_tiling & ETNA_LAYOUT_BIT_TILE;
82 bool src_super = rs->source_tiling & ETNA_LAYOUT_BIT_SUPER;
83 bool dst_super = rs->dest_tiling & ETNA_LAYOUT_BIT_SUPER;
84 bool src_multi = rs->source_tiling & ETNA_LAYOUT_BIT_MULTI;
85 bool dst_multi = rs->dest_tiling & ETNA_LAYOUT_BIT_MULTI;
86
87 /* Vivante RS needs widths to be a multiple of 16 or bad things
88 * happen, such as scribbing over memory, or the GPU hanging,
89 * even for non-tiled formats. As this is serious, use abort().
90 */
91 if (rs->width & ETNA_RS_WIDTH_MASK)
92 abort();
93
94 /* TODO could just pre-generate command buffer, would simply submit to one memcpy */
95 cs->RS_CONFIG = VIVS_RS_CONFIG_SOURCE_FORMAT(rs->source_format) |
96 COND(rs->downsample_x, VIVS_RS_CONFIG_DOWNSAMPLE_X) |
97 COND(rs->downsample_y, VIVS_RS_CONFIG_DOWNSAMPLE_Y) |
98 COND(src_tiled, VIVS_RS_CONFIG_SOURCE_TILED) |
99 VIVS_RS_CONFIG_DEST_FORMAT(rs->dest_format) |
100 COND(dst_tiled, VIVS_RS_CONFIG_DEST_TILED) |
101 COND(rs->swap_rb, VIVS_RS_CONFIG_SWAP_RB) |
102 COND(rs->flip, VIVS_RS_CONFIG_FLIP);
103
104 cs->RS_SOURCE_STRIDE = (rs->source_stride << source_stride_shift) |
105 COND(src_super, VIVS_RS_SOURCE_STRIDE_TILING) |
106 COND(src_multi, VIVS_RS_SOURCE_STRIDE_MULTI);
107
108 if (VIV_FEATURE(ctx->screen, ETNA_FEATURE_CACHE128B256BPERLINE))
109 cs->RS_SOURCE_STRIDE |= VIVS_RS_SOURCE_STRIDE_TS_MODE(rs->source_ts_mode) |
110 COND(src_super, VIVS_RS_SOURCE_STRIDE_SUPER_TILED_NEW);
111 else if ((rs->downsample_x || rs->downsample_y) && VIV_FEATURE(screen, ETNA_FEATURE_SMALL_MSAA))
112 cs->RS_SOURCE_STRIDE |= VIVS_RS_SOURCE_STRIDE_TS_MODE(TS_MODE_256B);
113
114 /* Initially all pipes are set to the base address of the source and
115 * destination buffer respectively. This will be overridden below as
116 * necessary for the multi-pipe, multi-tiled case.
117 */
118 for (unsigned pipe = 0; pipe < screen->specs.pixel_pipes; ++pipe) {
119 cs->source[pipe].bo = rs->source;
120 cs->source[pipe].offset = rs->source_offset;
121 cs->source[pipe].flags = ETNA_RELOC_READ;
122
123 cs->dest[pipe].bo = rs->dest;
124 cs->dest[pipe].offset = rs->dest_offset;
125 cs->dest[pipe].flags = ETNA_RELOC_WRITE;
126
127 cs->RS_PIPE_OFFSET[pipe] = VIVS_RS_PIPE_OFFSET_X(0) | VIVS_RS_PIPE_OFFSET_Y(0);
128 }
129
130 cs->RS_DEST_STRIDE = (rs->dest_stride << dest_stride_shift) |
131 COND(dst_super, VIVS_RS_DEST_STRIDE_TILING) |
132 COND(dst_multi, VIVS_RS_DEST_STRIDE_MULTI);
133
134 if (VIV_FEATURE(ctx->screen, ETNA_FEATURE_CACHE128B256BPERLINE))
135 cs->RS_DEST_STRIDE |= COND(dst_super, VIVS_RS_DEST_STRIDE_SUPER_TILED_NEW);
136
137 if (src_multi)
138 cs->source[1].offset = rs->source_offset + rs->source_stride * rs->source_padded_height / 2;
139
140 if (dst_multi)
141 cs->dest[1].offset = rs->dest_offset + rs->dest_stride * rs->dest_padded_height / 2;
142
143 cs->RS_WINDOW_SIZE = VIVS_RS_WINDOW_SIZE_WIDTH(rs->width) |
144 VIVS_RS_WINDOW_SIZE_HEIGHT(rs->height);
145
146 /* use dual pipe mode when required */
147 if (!screen->specs.single_buffer && screen->specs.pixel_pipes == 2 &&
148 !(rs->height & (rs->downsample_y ? 0xf : 0x7))) {
149 cs->RS_WINDOW_SIZE = VIVS_RS_WINDOW_SIZE_WIDTH(rs->width) |
150 VIVS_RS_WINDOW_SIZE_HEIGHT(rs->height / 2);
151 cs->RS_PIPE_OFFSET[1] = VIVS_RS_PIPE_OFFSET_X(0) | VIVS_RS_PIPE_OFFSET_Y(rs->height / 2);
152 }
153
154 cs->RS_DITHER[0] = rs->dither[0];
155 cs->RS_DITHER[1] = rs->dither[1];
156 cs->RS_CLEAR_CONTROL = VIVS_RS_CLEAR_CONTROL_BITS(rs->clear_bits) | rs->clear_mode;
157 cs->RS_FILL_VALUE[0] = rs->clear_value[0];
158 cs->RS_FILL_VALUE[1] = rs->clear_value[1];
159 cs->RS_FILL_VALUE[2] = rs->clear_value[2];
160 cs->RS_FILL_VALUE[3] = rs->clear_value[3];
161 cs->RS_EXTRA_CONFIG = VIVS_RS_EXTRA_CONFIG_AA(rs->aa) |
162 VIVS_RS_EXTRA_CONFIG_ENDIAN(rs->endian_mode);
163
164 /* If source the same as destination, and the hardware supports this,
165 * do an in-place resolve to fill in unrendered tiles.
166 */
167 if (screen->specs.single_buffer && rs->source == rs->dest &&
168 rs->source_offset == rs->dest_offset &&
169 rs->source_format == rs->dest_format &&
170 rs->source_tiling == rs->dest_tiling &&
171 src_super &&
172 rs->source_stride == rs->dest_stride &&
173 !rs->downsample_x && !rs->downsample_y &&
174 !rs->swap_rb && !rs->flip &&
175 !rs->clear_mode && rs->source_padded_width &&
176 !rs->source_ts_compressed) {
177 if (VIV_FEATURE(ctx->screen, ETNA_FEATURE_CACHE128B256BPERLINE))
178 cs->RS_EXTRA_CONFIG |= VIVS_RS_EXTRA_CONFIG_TS_MODE(rs->source_ts_mode);
179 /* Total number of tiles (same as for autodisable) */
180 cs->RS_KICKER_INPLACE = rs->tile_count;
181 }
182 cs->source_ts_valid = rs->source_ts_valid;
183 cs->valid = true;
184 }
185
186 /* modify the clear bits value in the compiled RS state */
187 static void
etna_modify_rs_clearbits(struct compiled_rs_state * cs,uint32_t clear_bits)188 etna_modify_rs_clearbits(struct compiled_rs_state *cs, uint32_t clear_bits)
189 {
190 cs->RS_CLEAR_CONTROL &= ~VIVS_RS_CLEAR_CONTROL_BITS__MASK;
191 cs->RS_CLEAR_CONTROL |= VIVS_RS_CLEAR_CONTROL_BITS(clear_bits);
192 }
193
194 #define EMIT_STATE(state_name, src_value) \
195 etna_coalsence_emit(stream, &coalesce, VIVS_##state_name, src_value)
196
197 #define EMIT_STATE_FIXP(state_name, src_value) \
198 etna_coalsence_emit_fixp(stream, &coalesce, VIVS_##state_name, src_value)
199
200 #define EMIT_STATE_RELOC(state_name, src_value) \
201 etna_coalsence_emit_reloc(stream, &coalesce, VIVS_##state_name, src_value)
202
203 /* submit RS state, without any processing and no dependence on context
204 * except TS if this is a source-to-destination blit. */
205 static void
etna_submit_rs_state(struct etna_context * ctx,const struct compiled_rs_state * cs)206 etna_submit_rs_state(struct etna_context *ctx,
207 const struct compiled_rs_state *cs)
208 {
209 struct etna_screen *screen = etna_screen(ctx->base.screen);
210 struct etna_cmd_stream *stream = ctx->stream;
211 struct etna_coalesce coalesce;
212
213 if (cs->RS_KICKER_INPLACE && !cs->source_ts_valid)
214 /* Inplace resolve is no-op if TS is not configured */
215 return;
216
217 ctx->stats.rs_operations++;
218
219 if (cs->RS_KICKER_INPLACE) {
220 etna_cmd_stream_reserve(stream, 6);
221 etna_coalesce_start(stream, &coalesce);
222 /* 0/1 */ EMIT_STATE(RS_EXTRA_CONFIG, cs->RS_EXTRA_CONFIG);
223 /* 2/3 */ EMIT_STATE(RS_SOURCE_STRIDE, cs->RS_SOURCE_STRIDE);
224 /* 4/5 */ EMIT_STATE(RS_KICKER_INPLACE, cs->RS_KICKER_INPLACE);
225 etna_coalesce_end(stream, &coalesce);
226 } else if (screen->specs.pixel_pipes > 1 ||
227 VIV_FEATURE(screen, ETNA_FEATURE_RS_NEW_BASEADDR)) {
228 etna_cmd_stream_reserve(stream, 34); /* worst case - both pipes multi=1 */
229 etna_coalesce_start(stream, &coalesce);
230 /* 0/1 */ EMIT_STATE(RS_CONFIG, cs->RS_CONFIG);
231 /* 2/3 */ EMIT_STATE(RS_SOURCE_STRIDE, cs->RS_SOURCE_STRIDE);
232 /* 4/5 */ EMIT_STATE(RS_DEST_STRIDE, cs->RS_DEST_STRIDE);
233 /* 6/7 */ EMIT_STATE_RELOC(RS_PIPE_SOURCE_ADDR(0), &cs->source[0]);
234 if (cs->RS_SOURCE_STRIDE & VIVS_RS_SOURCE_STRIDE_MULTI) {
235 /*8 */ EMIT_STATE_RELOC(RS_PIPE_SOURCE_ADDR(1), &cs->source[1]);
236 /*9 - pad */
237 }
238 /*10/11*/ EMIT_STATE_RELOC(RS_PIPE_DEST_ADDR(0), &cs->dest[0]);
239 if (cs->RS_DEST_STRIDE & VIVS_RS_DEST_STRIDE_MULTI) {
240 /*12*/ EMIT_STATE_RELOC(RS_PIPE_DEST_ADDR(1), &cs->dest[1]);
241 /*13 - pad */
242 }
243 /*14/15*/ EMIT_STATE(RS_PIPE_OFFSET(0), cs->RS_PIPE_OFFSET[0]);
244 /*16 */ EMIT_STATE(RS_PIPE_OFFSET(1), cs->RS_PIPE_OFFSET[1]);
245 /*17 - pad */
246 /*18/19*/ EMIT_STATE(RS_WINDOW_SIZE, cs->RS_WINDOW_SIZE);
247 /*20/21*/ EMIT_STATE(RS_DITHER(0), cs->RS_DITHER[0]);
248 /*22 */ EMIT_STATE(RS_DITHER(1), cs->RS_DITHER[1]);
249 /*23 - pad */
250 /*24/25*/ EMIT_STATE(RS_CLEAR_CONTROL, cs->RS_CLEAR_CONTROL);
251 /*26 */ EMIT_STATE(RS_FILL_VALUE(0), cs->RS_FILL_VALUE[0]);
252 /*27 */ EMIT_STATE(RS_FILL_VALUE(1), cs->RS_FILL_VALUE[1]);
253 /*28 */ EMIT_STATE(RS_FILL_VALUE(2), cs->RS_FILL_VALUE[2]);
254 /*29 */ EMIT_STATE(RS_FILL_VALUE(3), cs->RS_FILL_VALUE[3]);
255 /*30/31*/ EMIT_STATE(RS_EXTRA_CONFIG, cs->RS_EXTRA_CONFIG);
256 /*32/33*/ EMIT_STATE(RS_KICKER, 0xbeebbeeb);
257 etna_coalesce_end(stream, &coalesce);
258 } else {
259 etna_cmd_stream_reserve(stream, 22);
260 etna_coalesce_start(stream, &coalesce);
261 /* 0/1 */ EMIT_STATE(RS_CONFIG, cs->RS_CONFIG);
262 /* 2 */ EMIT_STATE_RELOC(RS_SOURCE_ADDR, &cs->source[0]);
263 /* 3 */ EMIT_STATE(RS_SOURCE_STRIDE, cs->RS_SOURCE_STRIDE);
264 /* 4 */ EMIT_STATE_RELOC(RS_DEST_ADDR, &cs->dest[0]);
265 /* 5 */ EMIT_STATE(RS_DEST_STRIDE, cs->RS_DEST_STRIDE);
266 /* 6/7 */ EMIT_STATE(RS_WINDOW_SIZE, cs->RS_WINDOW_SIZE);
267 /* 8/9 */ EMIT_STATE(RS_DITHER(0), cs->RS_DITHER[0]);
268 /*10 */ EMIT_STATE(RS_DITHER(1), cs->RS_DITHER[1]);
269 /*11 - pad */
270 /*12/13*/ EMIT_STATE(RS_CLEAR_CONTROL, cs->RS_CLEAR_CONTROL);
271 /*14 */ EMIT_STATE(RS_FILL_VALUE(0), cs->RS_FILL_VALUE[0]);
272 /*15 */ EMIT_STATE(RS_FILL_VALUE(1), cs->RS_FILL_VALUE[1]);
273 /*16 */ EMIT_STATE(RS_FILL_VALUE(2), cs->RS_FILL_VALUE[2]);
274 /*17 */ EMIT_STATE(RS_FILL_VALUE(3), cs->RS_FILL_VALUE[3]);
275 /*18/19*/ EMIT_STATE(RS_EXTRA_CONFIG, cs->RS_EXTRA_CONFIG);
276 /*20/21*/ EMIT_STATE(RS_KICKER, 0xbeebbeeb);
277 etna_coalesce_end(stream, &coalesce);
278 }
279 }
280
281 /* Generate clear command for a surface (non-fast clear case) */
282 static void
etna_rs_gen_clear_surface(struct etna_context * ctx,struct etna_surface * surf,uint64_t clear_value)283 etna_rs_gen_clear_surface(struct etna_context *ctx, struct etna_surface *surf,
284 uint64_t clear_value)
285 {
286 ASSERTED struct etna_screen *screen = ctx->screen;
287 struct etna_resource *dst = etna_resource(surf->base.texture);
288 uint32_t format;
289
290 switch (util_format_get_blocksizebits(surf->base.format)) {
291 case 16:
292 format = RS_FORMAT_A4R4G4B4;
293 break;
294 case 32:
295 format = RS_FORMAT_A8R8G8B8;
296 break;
297 case 64:
298 assert(screen->info->halti >= 2);
299 format = RS_FORMAT_64BPP_CLEAR;
300 break;
301 default:
302 unreachable("bpp not supported for clear by RS");
303 break;
304 }
305
306 /* use tiled clear if width is multiple of 16 */
307 bool tiled_clear = (surf->level->padded_width & ETNA_RS_WIDTH_MASK) == 0 &&
308 (surf->level->padded_height & ETNA_RS_HEIGHT_MASK) == 0;
309
310 etna_compile_rs_state( ctx, &surf->clear_command, &(struct rs_state) {
311 .source_format = format,
312 .dest_format = format,
313 .dest = dst->bo,
314 .dest_offset = surf->offset,
315 .dest_stride = surf->level->stride,
316 .dest_padded_height = surf->level->padded_height,
317 .dest_tiling = tiled_clear ? dst->layout : ETNA_LAYOUT_LINEAR,
318 .dither = {0xffffffff, 0xffffffff},
319 .width = surf->level->padded_width, /* These must be padded to 16x4 if !LINEAR, otherwise RS will hang */
320 .height = surf->level->padded_height,
321 .clear_value = {clear_value, clear_value >> 32, clear_value, clear_value >> 32},
322 .clear_mode = VIVS_RS_CLEAR_CONTROL_MODE_ENABLED1,
323 .clear_bits = 0xffff
324 });
325 }
326
327 static void
etna_blit_clear_color_rs(struct pipe_context * pctx,struct pipe_surface * dst,const union pipe_color_union * color)328 etna_blit_clear_color_rs(struct pipe_context *pctx, struct pipe_surface *dst,
329 const union pipe_color_union *color)
330 {
331 struct etna_context *ctx = etna_context(pctx);
332 struct etna_surface *surf = etna_surface(dst);
333 uint64_t new_clear_value = etna_clear_blit_pack_rgba(surf->base.format, color);
334
335 if (surf->level->ts_size) { /* TS: use precompiled clear command */
336 ctx->framebuffer.TS_COLOR_CLEAR_VALUE = new_clear_value;
337 ctx->framebuffer.TS_COLOR_CLEAR_VALUE_EXT = new_clear_value >> 32;
338
339 if (VIV_FEATURE(ctx->screen, ETNA_FEATURE_AUTO_DISABLE)) {
340 /* Set number of color tiles to be filled */
341 etna_set_state(ctx->stream, VIVS_TS_COLOR_AUTO_DISABLE_COUNT,
342 surf->level->padded_width * surf->level->padded_height / 16);
343 ctx->framebuffer.TS_MEM_CONFIG |= VIVS_TS_MEM_CONFIG_COLOR_AUTO_DISABLE;
344 }
345
346 /* update clear color in SW meta area of the buffer if TS is exported */
347 if (unlikely(new_clear_value != surf->level->clear_value &&
348 etna_resource_ext_ts(etna_resource(dst->texture))))
349 surf->level->ts_meta->v0.clear_value = new_clear_value;
350
351 assert(surf->ts_clear_command.valid);
352 etna_submit_rs_state(ctx, &surf->ts_clear_command);
353
354 etna_resource_level_ts_mark_valid(surf->level);
355 ctx->dirty |= ETNA_DIRTY_TS | ETNA_DIRTY_DERIVE_TS;
356 } else { /* Queue normal RS clear for non-TS surfaces */
357 /* If clear color changed or no valid command yet (re-)generate
358 * stored command */
359 if (unlikely(new_clear_value != surf->level->clear_value ||
360 !surf->clear_command.valid))
361 etna_rs_gen_clear_surface(ctx, surf, new_clear_value);
362
363 etna_submit_rs_state(ctx, &surf->clear_command);
364 }
365
366 surf->level->clear_value = new_clear_value;
367 resource_written(ctx, surf->base.texture);
368 etna_resource_level_mark_changed(surf->level);
369 }
370
371 static void
etna_blit_clear_zs_rs(struct pipe_context * pctx,struct pipe_surface * dst,unsigned buffers,double depth,unsigned stencil)372 etna_blit_clear_zs_rs(struct pipe_context *pctx, struct pipe_surface *dst,
373 unsigned buffers, double depth, unsigned stencil)
374 {
375 struct etna_context *ctx = etna_context(pctx);
376 struct etna_surface *surf = etna_surface(dst);
377 uint32_t new_clear_value = translate_clear_depth_stencil(surf->base.format, depth, stencil);
378 uint32_t new_clear_bits = 0, clear_bits_depth, clear_bits_stencil;
379
380 /* Get the channels to clear */
381 switch (surf->base.format) {
382 case PIPE_FORMAT_Z16_UNORM:
383 case PIPE_FORMAT_X8Z24_UNORM:
384 clear_bits_depth = 0xffff;
385 clear_bits_stencil = 0;
386 break;
387 case PIPE_FORMAT_S8_UINT_Z24_UNORM:
388 clear_bits_depth = 0xeeee;
389 clear_bits_stencil = 0x1111;
390 break;
391 default:
392 clear_bits_depth = clear_bits_stencil = 0xffff;
393 break;
394 }
395
396 if (buffers & PIPE_CLEAR_DEPTH)
397 new_clear_bits |= clear_bits_depth;
398 if (buffers & PIPE_CLEAR_STENCIL)
399 new_clear_bits |= clear_bits_stencil;
400
401 if (surf->level->ts_size && new_clear_bits == 0xffff) {
402 /* Set new clear depth value */
403 ctx->framebuffer.TS_DEPTH_CLEAR_VALUE = new_clear_value;
404 if (VIV_FEATURE(ctx->screen, ETNA_FEATURE_AUTO_DISABLE)) {
405 /* Set number of depth tiles to be filled */
406 etna_set_state(ctx->stream, VIVS_TS_DEPTH_AUTO_DISABLE_COUNT,
407 surf->level->padded_width * surf->level->padded_height / 16);
408 ctx->framebuffer.TS_MEM_CONFIG |= VIVS_TS_MEM_CONFIG_DEPTH_AUTO_DISABLE;
409 }
410
411 assert(surf->ts_clear_command.valid);
412 etna_submit_rs_state(ctx, &surf->ts_clear_command);
413
414 etna_resource_level_ts_mark_valid(surf->level);
415 ctx->dirty |= ETNA_DIRTY_TS;
416 } else { /* Queue normal RS clear for non-TS surfaces */
417 /* If the level has valid TS state we need to flush it, as the regular
418 * clear will not update the state and we must therefore invalidate it. */
419 etna_copy_resource(pctx, surf->base.texture, surf->base.texture,
420 surf->base.u.tex.level, surf->base.u.tex.level);
421
422 if (unlikely(new_clear_value != surf->level->clear_value ||
423 !surf->clear_command.valid)) {
424 /* If clear depth/stencil value changed or no valid command yet
425 * (re)-generate stored command */
426 etna_rs_gen_clear_surface(ctx, surf, new_clear_value);
427 }
428 /* Update the channels to be cleared */
429 etna_modify_rs_clearbits(&surf->clear_command, new_clear_bits);
430
431 etna_submit_rs_state(ctx, &surf->clear_command);
432
433 etna_resource_level_ts_mark_invalid(surf->level);
434 }
435
436 surf->level->clear_value = new_clear_value;
437 resource_written(ctx, surf->base.texture);
438 etna_resource_level_mark_changed(surf->level);
439 ctx->dirty |= ETNA_DIRTY_DERIVE_TS;
440 }
441
442 static void
etna_clear_rs(struct pipe_context * pctx,unsigned buffers,const struct pipe_scissor_state * scissor_state,const union pipe_color_union * color,double depth,unsigned stencil)443 etna_clear_rs(struct pipe_context *pctx, unsigned buffers, const struct pipe_scissor_state *scissor_state,
444 const union pipe_color_union *color, double depth, unsigned stencil)
445 {
446 struct etna_context *ctx = etna_context(pctx);
447
448 if (!etna_render_condition_check(pctx))
449 return;
450
451 /* Flush color and depth cache before clearing anything.
452 * This is especially important when coming from another surface, as
453 * otherwise it may clear part of the old surface instead. */
454 etna_set_state(ctx->stream, VIVS_GL_FLUSH_CACHE, VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
455 etna_stall(ctx->stream, SYNC_RECIPIENT_RA, SYNC_RECIPIENT_PE);
456
457 /* Preparation: Flush the TS if needed. This must be done after flushing
458 * color and depth, otherwise it can result in crashes */
459 bool need_ts_flush = false;
460 if ((buffers & PIPE_CLEAR_COLOR) && ctx->framebuffer_s.nr_cbufs) {
461 struct etna_surface *surf = etna_surface(ctx->framebuffer_s.cbufs[0]);
462
463 if (surf->level->ts_size)
464 need_ts_flush = true;
465 }
466 if ((buffers & PIPE_CLEAR_DEPTHSTENCIL) && ctx->framebuffer_s.zsbuf != NULL) {
467 struct etna_surface *surf = etna_surface(ctx->framebuffer_s.zsbuf);
468
469 if (surf->level->ts_size)
470 need_ts_flush = true;
471 }
472
473 if (need_ts_flush)
474 etna_set_state(ctx->stream, VIVS_TS_FLUSH_CACHE, VIVS_TS_FLUSH_CACHE_FLUSH);
475
476 /* No need to set up the TS here as RS clear operations (in contrast to
477 * resolve and copy) do not require the TS state.
478 */
479 if (buffers & PIPE_CLEAR_COLOR) {
480 for (int idx = 0; idx < ctx->framebuffer_s.nr_cbufs; ++idx) {
481 struct etna_surface *surf = etna_surface(ctx->framebuffer_s.cbufs[idx]);
482
483 etna_blit_clear_color_rs(pctx, ctx->framebuffer_s.cbufs[idx],
484 &color[idx]);
485
486 if (!etna_resource(surf->prsc)->explicit_flush)
487 etna_context_add_flush_resource(ctx, surf->prsc);
488 }
489 }
490
491 /* Flush the color and depth caches before each RS clear operation
492 * This fixes a hang on GC600. */
493 if (buffers & PIPE_CLEAR_DEPTHSTENCIL && buffers & PIPE_CLEAR_COLOR)
494 etna_set_state(ctx->stream, VIVS_GL_FLUSH_CACHE,
495 VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
496
497 if ((buffers & PIPE_CLEAR_DEPTHSTENCIL) && ctx->framebuffer_s.zsbuf != NULL)
498 etna_blit_clear_zs_rs(pctx, ctx->framebuffer_s.zsbuf, buffers, depth, stencil);
499
500 etna_stall(ctx->stream, SYNC_RECIPIENT_RA, SYNC_RECIPIENT_PE);
501 }
502
503 static bool
etna_manual_blit(struct etna_resource * dst,struct etna_resource_level * dst_lev,unsigned int dst_offset,struct etna_resource * src,struct etna_resource_level * src_lev,unsigned int src_offset,const struct pipe_blit_info * blit_info)504 etna_manual_blit(struct etna_resource *dst, struct etna_resource_level *dst_lev,
505 unsigned int dst_offset, struct etna_resource *src,
506 struct etna_resource_level *src_lev, unsigned int src_offset,
507 const struct pipe_blit_info *blit_info)
508 {
509 void *smap, *srow, *dmap, *drow;
510 size_t tile_size;
511
512 assert(src->layout == ETNA_LAYOUT_TILED);
513 assert(dst->layout == ETNA_LAYOUT_TILED);
514 assert(src->base.nr_samples == 0);
515 assert(dst->base.nr_samples == 0);
516
517 tile_size = util_format_get_blocksize(blit_info->src.format) * 4 * 4;
518
519 smap = etna_bo_map(src->bo);
520 if (!smap)
521 return false;
522
523 dmap = etna_bo_map(dst->bo);
524 if (!dmap)
525 return false;
526
527 srow = smap + src_offset;
528 drow = dmap + dst_offset;
529
530 etna_bo_cpu_prep(src->bo, DRM_ETNA_PREP_READ);
531 etna_bo_cpu_prep(dst->bo, DRM_ETNA_PREP_WRITE);
532
533 for (int y = 0; y < blit_info->src.box.height; y += 4) {
534 memcpy(drow, srow, tile_size * blit_info->src.box.width);
535 srow += src_lev->stride * 4;
536 drow += dst_lev->stride * 4;
537 }
538
539 etna_bo_cpu_fini(dst->bo);
540 etna_bo_cpu_fini(src->bo);
541
542 return true;
543 }
544
545 static inline size_t
etna_compute_tileoffset(const struct pipe_box * box,enum pipe_format format,size_t stride,enum etna_surface_layout layout)546 etna_compute_tileoffset(const struct pipe_box *box, enum pipe_format format,
547 size_t stride, enum etna_surface_layout layout)
548 {
549 size_t offset;
550 unsigned int x = box->x, y = box->y;
551 unsigned int blocksize = util_format_get_blocksize(format);
552
553 switch (layout) {
554 case ETNA_LAYOUT_LINEAR:
555 offset = y * stride + x * blocksize;
556 break;
557 case ETNA_LAYOUT_MULTI_TILED:
558 y >>= 1;
559 FALLTHROUGH;
560 case ETNA_LAYOUT_TILED:
561 assert(!(x & 0x03) && !(y & 0x03));
562 offset = (y & ~0x03) * stride + blocksize * ((x & ~0x03) << 2);
563 break;
564 case ETNA_LAYOUT_MULTI_SUPERTILED:
565 y >>= 1;
566 FALLTHROUGH;
567 case ETNA_LAYOUT_SUPER_TILED:
568 assert(!(x & 0x3f) && !(y & 0x3f));
569 offset = (y & ~0x3f) * stride + blocksize * ((x & ~0x3f) << 6);
570 break;
571 default:
572 unreachable("invalid resource layout");
573 }
574
575 return offset;
576 }
577
578 static inline void
etna_get_rs_alignment_mask(const struct etna_context * ctx,const enum etna_surface_layout layout,unsigned int * width_mask,unsigned int * height_mask)579 etna_get_rs_alignment_mask(const struct etna_context *ctx,
580 const enum etna_surface_layout layout,
581 unsigned int *width_mask, unsigned int *height_mask)
582 {
583 struct etna_screen *screen = ctx->screen;
584 unsigned int h_align, w_align;
585
586 if (layout & ETNA_LAYOUT_BIT_SUPER) {
587 w_align = 64;
588 h_align = 64 * screen->specs.pixel_pipes;
589 } else {
590 w_align = ETNA_RS_WIDTH_MASK + 1;
591 h_align = ETNA_RS_HEIGHT_MASK + 1;
592 }
593
594 *width_mask = w_align - 1;
595 *height_mask = h_align -1;
596 }
597
598 static bool
etna_try_rs_blit(struct pipe_context * pctx,const struct pipe_blit_info * blit_info)599 etna_try_rs_blit(struct pipe_context *pctx,
600 const struct pipe_blit_info *blit_info)
601 {
602 struct etna_context *ctx = etna_context(pctx);
603 struct etna_resource *src = etna_resource(blit_info->src.resource);
604 struct etna_resource *dst = etna_resource(blit_info->dst.resource);
605 struct compiled_rs_state copy_to_screen;
606 int src_xscale, src_yscale, dst_xscale, dst_yscale;
607 bool downsample_x = false, downsample_y = false;
608
609 /* Ensure that the level is valid */
610 assert(blit_info->src.level <= src->base.last_level);
611 assert(blit_info->dst.level <= dst->base.last_level);
612
613 if (!translate_samples_to_xyscale(src->base.nr_samples, &src_xscale, &src_yscale))
614 return false;
615 if (!translate_samples_to_xyscale(dst->base.nr_samples, &dst_xscale, &dst_yscale))
616 return false;
617
618 /* RS does not support upscaling */
619 if ((src_xscale < dst_xscale) || (src_yscale < dst_yscale))
620 return false;
621
622 if (src_xscale > dst_xscale)
623 downsample_x = true;
624 if (src_yscale > dst_yscale)
625 downsample_y = true;
626
627 /* The width/height are in pixels; they do not change as a result of
628 * multi-sampling. So, when blitting from a 4x multisampled surface
629 * to a non-multisampled surface, the width and height will be
630 * identical. As we do not support scaling, reject different sizes. */
631 if (blit_info->dst.box.width != blit_info->src.box.width ||
632 blit_info->dst.box.height != blit_info->src.box.height) {
633 DBG("scaling requested: source %dx%d destination %dx%d",
634 blit_info->src.box.width, blit_info->src.box.height,
635 blit_info->dst.box.width, blit_info->dst.box.height);
636 return false;
637 }
638
639 /* No masks - RS can't copy specific channels */
640 unsigned mask = util_format_get_mask(blit_info->dst.format);
641 if ((blit_info->mask & mask) != mask) {
642 DBG("sub-mask requested: 0x%02x vs format mask 0x%02x", blit_info->mask, mask);
643 return false;
644 }
645
646 /* Only support same format (used tiling/detiling) blits for now.
647 * TODO: figure out which different-format blits are possible and test them
648 * - fail if swizzle needed
649 * - avoid trying to convert between float/int formats?
650 */
651 if (blit_info->src.format != blit_info->dst.format)
652 return false;
653
654 /* try to find a exact format match first */
655 uint32_t format = translate_rs_format(blit_info->dst.format);
656 /* When not resolving MSAA, but only doing a layout conversion, we can get
657 * away with a fallback format of matching size.
658 */
659 if (format == ETNA_NO_MATCH && !downsample_x && !downsample_y)
660 format = etna_compatible_rs_format(blit_info->dst.format);
661 if (format == ETNA_NO_MATCH)
662 return false;
663
664 if (blit_info->scissor_enable ||
665 blit_info->dst.box.depth != blit_info->src.box.depth ||
666 blit_info->dst.box.depth != 1) {
667 return false;
668 }
669
670 unsigned w_mask, h_mask;
671
672 etna_get_rs_alignment_mask(ctx, src->layout, &w_mask, &h_mask);
673 if ((blit_info->src.box.x & w_mask) || (blit_info->src.box.y & h_mask))
674 return false;
675
676 etna_get_rs_alignment_mask(ctx, dst->layout, &w_mask, &h_mask);
677 if ((blit_info->dst.box.x & w_mask) || (blit_info->dst.box.y & h_mask))
678 return false;
679
680 struct etna_resource_level *src_lev = &src->levels[blit_info->src.level];
681 struct etna_resource_level *dst_lev = &dst->levels[blit_info->dst.level];
682
683 /* we may be given coordinates up to the padded width to avoid
684 * any alignment issues with different tiling formats */
685 assert((blit_info->src.box.x + blit_info->src.box.width) * src_xscale <= src_lev->padded_width);
686 assert((blit_info->src.box.y + blit_info->src.box.height) * src_yscale <= src_lev->padded_height);
687 assert(blit_info->dst.box.x + blit_info->dst.box.width <= dst_lev->padded_width);
688 assert(blit_info->dst.box.y + blit_info->dst.box.height <= dst_lev->padded_height);
689
690 unsigned src_offset = src_lev->offset +
691 blit_info->src.box.z * src_lev->layer_stride +
692 etna_compute_tileoffset(&blit_info->src.box,
693 blit_info->src.format,
694 src_lev->stride,
695 src->layout);
696 unsigned dst_offset = dst_lev->offset +
697 blit_info->dst.box.z * dst_lev->layer_stride +
698 etna_compute_tileoffset(&blit_info->dst.box,
699 blit_info->dst.format,
700 dst_lev->stride,
701 dst->layout);
702
703 if (src_lev->padded_width <= ETNA_RS_WIDTH_MASK ||
704 dst_lev->padded_width <= ETNA_RS_WIDTH_MASK ||
705 src_lev->padded_height <= ETNA_RS_HEIGHT_MASK ||
706 dst_lev->padded_height <= ETNA_RS_HEIGHT_MASK)
707 goto manual;
708
709 /* If the width is not aligned to the RS width, but is within our
710 * padding, adjust the width to suite the RS width restriction.
711 * Note: the RS width/height are converted to source samples here. */
712 unsigned int width = blit_info->src.box.width * src_xscale;
713 unsigned int height = blit_info->src.box.height * src_yscale;
714 unsigned int w_align = (ETNA_RS_WIDTH_MASK + 1) * src_xscale;
715 unsigned int h_align = (ETNA_RS_HEIGHT_MASK + 1) * src_yscale;
716
717 if (width & (w_align - 1) && width >= src_lev->width * src_xscale && width >= dst_lev->width)
718 width = align(width, w_align);
719
720 if (height & (h_align - 1) && height >= src_lev->height * src_yscale && height >= dst_lev->height) {
721 height = align(height, h_align);
722
723 /* Try to increase alignment to multi-pipe requirements to unlock
724 * multi-pipe resolve for increased performance. */
725 if (!ctx->screen->specs.single_buffer) {
726 unsigned int pipe_align = align(height, h_align * ctx->screen->specs.pixel_pipes);
727
728 if (pipe_align <= src_lev->padded_height &&
729 pipe_align <= dst_lev->padded_height * src_yscale)
730 height = pipe_align;
731 }
732 }
733
734 /* The padded dimensions are in samples */
735 if (width > src_lev->padded_width ||
736 width > dst_lev->padded_width * src_xscale ||
737 height > src_lev->padded_height ||
738 height > dst_lev->padded_height * src_yscale ||
739 width & (w_align - 1) || height & (h_align - 1))
740 goto manual;
741
742 /* Flush destination, as the blit will invalidate any pending TS changes. */
743 if (dst != src && etna_resource_level_needs_flush(dst_lev))
744 etna_copy_resource(pctx, &dst->base, &dst->base,
745 blit_info->dst.level, blit_info->dst.level);
746
747 /* Always flush color and depth cache together before resolving. This makes
748 * sure that all previous cache content written by the PE is flushed out
749 * before RS uses the pixel pipes, which invalidates those caches. */
750 etna_set_state(ctx->stream, VIVS_GL_FLUSH_CACHE,
751 VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
752 etna_stall(ctx->stream, SYNC_RECIPIENT_RA, SYNC_RECIPIENT_PE);
753
754 /* Set up color TS to source surface before blit, if needed */
755 bool source_ts_valid = false;
756 if (etna_resource_level_ts_valid(src_lev)) {
757 struct etna_reloc reloc;
758 unsigned ts_offset =
759 src_lev->ts_offset + blit_info->src.box.z * src_lev->ts_layer_stride;
760 uint32_t ts_mem_config = 0;
761
762 /* flush TS cache before changing to another TS configuration */
763 etna_set_state(ctx->stream, VIVS_TS_FLUSH_CACHE, VIVS_TS_FLUSH_CACHE_FLUSH);
764
765 if (src_lev->ts_compress_fmt >= 0) {
766 ts_mem_config |= VIVS_TS_MEM_CONFIG_COLOR_COMPRESSION |
767 VIVS_TS_MEM_CONFIG_COLOR_COMPRESSION_FORMAT(src_lev->ts_compress_fmt);
768 }
769
770 etna_set_state(ctx->stream, VIVS_TS_MEM_CONFIG,
771 VIVS_TS_MEM_CONFIG_COLOR_FAST_CLEAR | ts_mem_config);
772
773 memset(&reloc, 0, sizeof(struct etna_reloc));
774 reloc.bo = src->ts_bo;
775 reloc.offset = ts_offset;
776 reloc.flags = ETNA_RELOC_READ;
777 etna_set_state_reloc(ctx->stream, VIVS_TS_COLOR_STATUS_BASE, &reloc);
778
779 memset(&reloc, 0, sizeof(struct etna_reloc));
780 reloc.bo = src->bo;
781 reloc.offset = src_lev->offset +
782 blit_info->src.box.z * src_lev->layer_stride;
783 reloc.flags = ETNA_RELOC_READ;
784 etna_set_state_reloc(ctx->stream, VIVS_TS_COLOR_SURFACE_BASE, &reloc);
785
786 etna_set_state(ctx->stream, VIVS_TS_COLOR_CLEAR_VALUE, src_lev->clear_value);
787 etna_set_state(ctx->stream, VIVS_TS_COLOR_CLEAR_VALUE_EXT, src_lev->clear_value >> 32);
788
789 source_ts_valid = true;
790 } else {
791 etna_set_state(ctx->stream, VIVS_TS_MEM_CONFIG, 0);
792 }
793 ctx->dirty |= ETNA_DIRTY_TS;
794
795 /* Kick off RS here */
796 etna_compile_rs_state(ctx, ©_to_screen, &(struct rs_state) {
797 .source_format = format,
798 .source_tiling = src->layout,
799 .source = src->bo,
800 .source_offset = src_offset,
801 .source_stride = src_lev->stride,
802 .source_padded_width = src_lev->padded_width,
803 .source_padded_height = src_lev->padded_height,
804 .source_ts_valid = source_ts_valid,
805 .source_ts_mode = src_lev->ts_mode,
806 .source_ts_compressed = src_lev->ts_compress_fmt >= 0,
807 .dest_format = format,
808 .dest_tiling = dst->layout,
809 .dest = dst->bo,
810 .dest_offset = dst_offset,
811 .dest_stride = dst_lev->stride,
812 .dest_padded_height = dst_lev->padded_height,
813 .downsample_x = downsample_x,
814 .downsample_y = downsample_y,
815 .swap_rb = translate_rb_src_dst_swap(src->base.format, dst->base.format),
816 .dither = {0xffffffff, 0xffffffff}, // XXX dither when going from 24 to 16 bit?
817 .clear_mode = VIVS_RS_CLEAR_CONTROL_MODE_DISABLED,
818 .width = width,
819 .height = height,
820 .tile_count = src_lev->layer_stride /
821 etna_screen_get_tile_size(ctx->screen, src_lev->ts_mode,
822 src->base.nr_samples > 1),
823 });
824
825 etna_submit_rs_state(ctx, ©_to_screen);
826 resource_read(ctx, &src->base);
827 resource_written(ctx, &dst->base);
828 etna_resource_level_mark_changed(dst_lev);
829
830 /* We don't need to mark the TS as invalid if this was just a flush without
831 * compression, as in that case only clear tiles are filled and the tile
832 * status still matches the blit target buffer. For compressed formats the
833 * tiles are decompressed, so tile status doesn't match anymore.
834 */
835 if (src != dst || src_lev->ts_compress_fmt >= 0)
836 etna_resource_level_ts_mark_invalid(dst_lev);
837
838 ctx->dirty |= ETNA_DIRTY_DERIVE_TS;
839
840 return true;
841
842 manual:
843 if (src->layout == ETNA_LAYOUT_TILED && dst->layout == ETNA_LAYOUT_TILED) {
844 if ((etna_resource_status(ctx, src) & ETNA_PENDING_WRITE) ||
845 (etna_resource_status(ctx, dst) & ETNA_PENDING_WRITE))
846 etna_flush(pctx, NULL, 0, true);
847
848 perf_debug_ctx(ctx, "RS blit falls back to sw");
849
850 return etna_manual_blit(dst, dst_lev, dst_offset, src, src_lev, src_offset, blit_info);
851 }
852
853 return false;
854 }
855
856 void
etna_clear_blit_rs_init(struct pipe_context * pctx)857 etna_clear_blit_rs_init(struct pipe_context *pctx)
858 {
859 struct etna_context *ctx = etna_context(pctx);
860
861 DBG("etnaviv: Using RS blit engine");
862 pctx->clear = etna_clear_rs;
863 ctx->blit = etna_try_rs_blit;
864 }
865