Lines Matching +full:cs +full:- +full:2
1 // SPDX-License-Identifier: MIT
24 * The per-platform tables are u8-encoded in @data. Decode @data and set the
26 * for each byte. There are 2 steps: decoding commands and decoding addresses.
29 * [7]: create NOPs - number of NOPs are set in lower bits
54 #define REG(x) (((x) >> 2) | BUILD_BUG_ON_ZERO(x >= 0x200)) in set_offsets()
57 (((x) >> 2) & 0x7f) in set_offsets()
60 const u32 base = engine->mmio_base; in set_offsets()
78 if (GRAPHICS_VER(engine->i915) >= 11) in set_offsets()
93 regs[0] = base + (offset << 2); in set_offsets()
94 regs += 2; in set_offsets()
95 } while (--count); in set_offsets()
101 if (GRAPHICS_VER(engine->i915) >= 11) in set_offsets()
134 LRI(2, 0),
623 NOP(2),
624 LRI(2, POSTED),
649 GEM_BUG_ON(GRAPHICS_VER(engine->i915) >= 12 && in reg_offsets()
652 if (engine->flags & I915_ENGINE_HAS_RCS_REG_STATE) { in reg_offsets()
653 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 70)) in reg_offsets()
655 else if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) in reg_offsets()
657 else if (GRAPHICS_VER(engine->i915) >= 12) in reg_offsets()
659 else if (GRAPHICS_VER(engine->i915) >= 11) in reg_offsets()
661 else if (GRAPHICS_VER(engine->i915) >= 9) in reg_offsets()
666 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) in reg_offsets()
668 else if (GRAPHICS_VER(engine->i915) >= 12) in reg_offsets()
670 else if (GRAPHICS_VER(engine->i915) >= 9) in reg_offsets()
679 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) in lrc_ring_mi_mode()
681 else if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_mi_mode()
683 else if (GRAPHICS_VER(engine->i915) >= 9) in lrc_ring_mi_mode()
685 else if (engine->class == RENDER_CLASS) in lrc_ring_mi_mode()
688 return -1; in lrc_ring_mi_mode()
693 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) in lrc_ring_bb_offset()
695 else if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_bb_offset()
697 else if (GRAPHICS_VER(engine->i915) >= 9) in lrc_ring_bb_offset()
699 else if (GRAPHICS_VER(engine->i915) >= 8 && in lrc_ring_bb_offset()
700 engine->class == RENDER_CLASS) in lrc_ring_bb_offset()
703 return -1; in lrc_ring_bb_offset()
708 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) in lrc_ring_gpr0()
710 else if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_gpr0()
712 else if (GRAPHICS_VER(engine->i915) >= 9) in lrc_ring_gpr0()
714 else if (engine->class == RENDER_CLASS) in lrc_ring_gpr0()
717 return -1; in lrc_ring_gpr0()
722 if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_wa_bb_per_ctx()
724 else if (GRAPHICS_VER(engine->i915) >= 9 || engine->class == RENDER_CLASS) in lrc_ring_wa_bb_per_ctx()
727 return -1; in lrc_ring_wa_bb_per_ctx()
738 return x + 2; in lrc_ring_indirect_ptr()
749 return x + 2; in lrc_ring_indirect_offset()
755 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) in lrc_ring_cmd_buf_cctl()
761 else if (engine->class != RENDER_CLASS) in lrc_ring_cmd_buf_cctl()
762 return -1; in lrc_ring_cmd_buf_cctl()
763 else if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_cmd_buf_cctl()
765 else if (GRAPHICS_VER(engine->i915) >= 11) in lrc_ring_cmd_buf_cctl()
768 return -1; in lrc_ring_cmd_buf_cctl()
774 if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_indirect_offset_default()
776 else if (GRAPHICS_VER(engine->i915) >= 11) in lrc_ring_indirect_offset_default()
778 else if (GRAPHICS_VER(engine->i915) >= 9) in lrc_ring_indirect_offset_default()
780 else if (GRAPHICS_VER(engine->i915) >= 8) in lrc_ring_indirect_offset_default()
783 GEM_BUG_ON(GRAPHICS_VER(engine->i915) < 8); in lrc_ring_indirect_offset_default()
793 GEM_BUG_ON(lrc_ring_wa_bb_per_ctx(engine) == -1); in lrc_setup_bb_per_ctx()
808 GEM_BUG_ON(lrc_ring_indirect_ptr(engine) == -1); in lrc_setup_indirect_ctx()
812 GEM_BUG_ON(lrc_ring_indirect_offset(engine) == -1); in lrc_setup_indirect_ctx()
823 * Wa_14019159160 - Case 2. in ctx_needs_runalone()
825 * the LRC run-alone bit or else the encryption/decryption will not happen. in ctx_needs_runalone()
826 * NOTE: Case 2 only applies to PXP use-case of said workaround. in ctx_needs_runalone()
828 if (GRAPHICS_VER_FULL(ce->engine->i915) >= IP_VER(12, 70) && in ctx_needs_runalone()
829 (ce->engine->class == COMPUTE_CLASS || ce->engine->class == RENDER_CLASS)) { in ctx_needs_runalone()
831 gem_ctx = rcu_dereference(ce->gem_context); in ctx_needs_runalone()
833 ctx_is_protected = gem_ctx->uses_protected_content; in ctx_needs_runalone()
852 if (GRAPHICS_VER(engine->i915) < 11) in init_common_regs()
855 /* Wa_14019159160 - Case 2.*/ in init_common_regs()
860 regs[CTX_TIMESTAMP] = ce->stats.runtime.last; in init_common_regs()
863 if (loc != -1) in init_common_regs()
870 const struct i915_ctx_workarounds * const wa_ctx = &engine->wa_ctx; in init_wa_bb_regs()
872 if (wa_ctx->per_ctx.size) { in init_wa_bb_regs()
873 const u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma); in init_wa_bb_regs()
875 GEM_BUG_ON(lrc_ring_wa_bb_per_ctx(engine) == -1); in init_wa_bb_regs()
877 (ggtt_offset + wa_ctx->per_ctx.offset) | 0x01; in init_wa_bb_regs()
880 if (wa_ctx->indirect_ctx.size) { in init_wa_bb_regs()
882 i915_ggtt_offset(wa_ctx->vma) + in init_wa_bb_regs()
883 wa_ctx->indirect_ctx.offset, in init_wa_bb_regs()
884 wa_ctx->indirect_ctx.size); in init_wa_bb_regs()
890 if (i915_vm_is_4lvl(&ppgtt->vm)) { in init_ppgtt_regs()
898 ASSIGN_CTX_PDP(ppgtt, regs, 2); in init_ppgtt_regs()
907 return i915_vm_to_ggtt(vm)->alias; in vm_alias()
917 if (x != -1) { in __reset_stop_ring()
945 init_ppgtt_regs(regs, vm_alias(ce->vm)); in __lrc_init_regs()
956 __lrc_init_regs(ce->lrc_reg_state, ce, engine, inhibit); in lrc_init_regs()
962 __reset_stop_ring(ce->lrc_reg_state, engine); in lrc_reset_regs()
971 vaddr += engine->context_size; in set_redzone()
982 vaddr += engine->context_size; in check_redzone()
985 drm_err_once(&engine->i915->drm, in check_redzone()
987 engine->name); in check_redzone()
992 return PAGE_SIZE * ce->wa_bb_page; in context_wa_bb_offset()
1005 GEM_BUG_ON(!ce->wa_bb_page); in context_wabb()
1007 ptr = ce->lrc_reg_state; in context_wabb()
1008 ptr -= LRC_STATE_OFFSET; /* back to start of context image */ in context_wabb()
1023 if (ce->default_state) { in lrc_init_state()
1024 shmem_read(ce->default_state, 0, state, engine->context_size); in lrc_init_state()
1025 __set_bit(CONTEXT_VALID_BIT, &ce->flags); in lrc_init_state()
1029 /* Clear the ppHWSP (inc. per-context counters) */ in lrc_init_state()
1033 if (ce->wa_bb_page) in lrc_init_state()
1045 return i915_ggtt_offset(ce->state) + context_wa_bb_offset(ce); in lrc_indirect_bb()
1048 static u32 *setup_predicate_disable_wa(const struct intel_context *ce, u32 *cs) in setup_predicate_disable_wa() argument
1051 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT | (4 - 2); in setup_predicate_disable_wa()
1052 *cs++ = lrc_indirect_bb(ce) + DG2_PREDICATE_RESULT_WA; in setup_predicate_disable_wa()
1053 *cs++ = 0; in setup_predicate_disable_wa()
1054 *cs++ = 0; /* No predication */ in setup_predicate_disable_wa()
1057 *cs++ = MI_BATCH_BUFFER_END | BIT(15); in setup_predicate_disable_wa()
1058 *cs++ = MI_SET_PREDICATE | MI_SET_PREDICATE_DISABLE; in setup_predicate_disable_wa()
1061 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT | (4 - 2); in setup_predicate_disable_wa()
1062 *cs++ = lrc_indirect_bb(ce) + DG2_PREDICATE_RESULT_WA; in setup_predicate_disable_wa()
1063 *cs++ = 0; in setup_predicate_disable_wa()
1064 *cs++ = 1; /* enable predication before the next BB */ in setup_predicate_disable_wa()
1066 *cs++ = MI_BATCH_BUFFER_END; in setup_predicate_disable_wa()
1067 GEM_BUG_ON(offset_in_page(cs) > DG2_PREDICATE_RESULT_WA); in setup_predicate_disable_wa()
1069 return cs; in setup_predicate_disable_wa()
1079 context_size = round_up(engine->context_size, I915_GTT_PAGE_SIZE); in __lrc_alloc_state()
1084 if (GRAPHICS_VER(engine->i915) >= 12) { in __lrc_alloc_state()
1085 ce->wa_bb_page = context_size / PAGE_SIZE; in __lrc_alloc_state()
1087 context_size += PAGE_SIZE * 2; in __lrc_alloc_state()
1091 ce->parallel.guc.parent_page = context_size / PAGE_SIZE; in __lrc_alloc_state()
1095 obj = i915_gem_object_create_lmem(engine->i915, context_size, in __lrc_alloc_state()
1098 obj = i915_gem_object_create_shmem(engine->i915, context_size); in __lrc_alloc_state()
1105 * index 2) on GPU side. in __lrc_alloc_state()
1107 if (intel_gt_needs_wa_22016122933(engine->gt)) in __lrc_alloc_state()
1111 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); in __lrc_alloc_state()
1123 struct intel_timeline *tl = fetch_and_zero(&ce->timeline); in pinned_timeline()
1134 GEM_BUG_ON(ce->state); in lrc_alloc()
1137 ce->default_state = engine->default_state; in lrc_alloc()
1143 ring = intel_engine_create_ring(engine, ce->ring_size); in lrc_alloc()
1149 if (!page_mask_bits(ce->timeline)) { in lrc_alloc()
1156 if (unlikely(ce->timeline)) in lrc_alloc()
1159 tl = intel_timeline_create(engine->gt); in lrc_alloc()
1165 ce->timeline = tl; in lrc_alloc()
1168 ce->ring = ring; in lrc_alloc()
1169 ce->state = vma; in lrc_alloc()
1184 intel_ring_reset(ce->ring, ce->ring->emit); in lrc_reset()
1187 lrc_init_regs(ce, ce->engine, true); in lrc_reset()
1188 ce->lrc.lrca = lrc_update_regs(ce, ce->engine, ce->ring->tail); in lrc_reset()
1197 GEM_BUG_ON(!ce->state); in lrc_pre_pin()
1198 GEM_BUG_ON(!i915_vma_is_pinned(ce->state)); in lrc_pre_pin()
1200 *vaddr = i915_gem_object_pin_map(ce->state->obj, in lrc_pre_pin()
1201 intel_gt_coherent_map_type(ce->engine->gt, in lrc_pre_pin()
1202 ce->state->obj, in lrc_pre_pin()
1214 ce->lrc_reg_state = vaddr + LRC_STATE_OFFSET; in lrc_pin()
1216 if (!__test_and_set_bit(CONTEXT_INIT_BIT, &ce->flags)) in lrc_pin()
1219 ce->lrc.lrca = lrc_update_regs(ce, engine, ce->ring->tail); in lrc_pin()
1225 if (unlikely(ce->parallel.last_rq)) { in lrc_unpin()
1226 i915_request_put(ce->parallel.last_rq); in lrc_unpin()
1227 ce->parallel.last_rq = NULL; in lrc_unpin()
1229 check_redzone((void *)ce->lrc_reg_state - LRC_STATE_OFFSET, in lrc_unpin()
1230 ce->engine); in lrc_unpin()
1235 i915_gem_object_unpin_map(ce->state->obj); in lrc_post_unpin()
1240 if (!ce->state) in lrc_fini()
1243 intel_ring_put(fetch_and_zero(&ce->ring)); in lrc_fini()
1244 i915_vma_put(fetch_and_zero(&ce->state)); in lrc_fini()
1251 GEM_BUG_ON(!i915_active_is_idle(&ce->active)); in lrc_destroy()
1261 gen12_emit_timestamp_wa(const struct intel_context *ce, u32 *cs) in gen12_emit_timestamp_wa() argument
1263 *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | in gen12_emit_timestamp_wa()
1266 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_timestamp_wa()
1267 *cs++ = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET + in gen12_emit_timestamp_wa()
1269 *cs++ = 0; in gen12_emit_timestamp_wa()
1271 *cs++ = MI_LOAD_REGISTER_REG | in gen12_emit_timestamp_wa()
1274 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_timestamp_wa()
1275 *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(0)); in gen12_emit_timestamp_wa()
1277 *cs++ = MI_LOAD_REGISTER_REG | in gen12_emit_timestamp_wa()
1280 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_timestamp_wa()
1281 *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(0)); in gen12_emit_timestamp_wa()
1283 return cs; in gen12_emit_timestamp_wa()
1287 gen12_emit_restore_scratch(const struct intel_context *ce, u32 *cs) in gen12_emit_restore_scratch() argument
1289 GEM_BUG_ON(lrc_ring_gpr0(ce->engine) == -1); in gen12_emit_restore_scratch()
1291 *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | in gen12_emit_restore_scratch()
1294 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_restore_scratch()
1295 *cs++ = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET + in gen12_emit_restore_scratch()
1296 (lrc_ring_gpr0(ce->engine) + 1) * sizeof(u32); in gen12_emit_restore_scratch()
1297 *cs++ = 0; in gen12_emit_restore_scratch()
1299 return cs; in gen12_emit_restore_scratch()
1303 gen12_emit_cmd_buf_wa(const struct intel_context *ce, u32 *cs) in gen12_emit_cmd_buf_wa() argument
1305 GEM_BUG_ON(lrc_ring_cmd_buf_cctl(ce->engine) == -1); in gen12_emit_cmd_buf_wa()
1307 *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | in gen12_emit_cmd_buf_wa()
1310 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_cmd_buf_wa()
1311 *cs++ = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET + in gen12_emit_cmd_buf_wa()
1312 (lrc_ring_cmd_buf_cctl(ce->engine) + 1) * sizeof(u32); in gen12_emit_cmd_buf_wa()
1313 *cs++ = 0; in gen12_emit_cmd_buf_wa()
1315 *cs++ = MI_LOAD_REGISTER_REG | in gen12_emit_cmd_buf_wa()
1318 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_cmd_buf_wa()
1319 *cs++ = i915_mmio_reg_offset(RING_CMD_BUF_CCTL(0)); in gen12_emit_cmd_buf_wa()
1321 return cs; in gen12_emit_cmd_buf_wa()
1332 dg2_emit_draw_watermark_setting(u32 *cs) in dg2_emit_draw_watermark_setting() argument
1334 *cs++ = MI_LOAD_REGISTER_IMM(1); in dg2_emit_draw_watermark_setting()
1335 *cs++ = i915_mmio_reg_offset(DRAW_WATERMARK); in dg2_emit_draw_watermark_setting()
1336 *cs++ = REG_FIELD_PREP(VERT_WM_VAL, 0x3FF); in dg2_emit_draw_watermark_setting()
1338 return cs; in dg2_emit_draw_watermark_setting()
1342 gen12_invalidate_state_cache(u32 *cs) in gen12_invalidate_state_cache() argument
1344 *cs++ = MI_LOAD_REGISTER_IMM(1); in gen12_invalidate_state_cache()
1345 *cs++ = i915_mmio_reg_offset(GEN12_CS_DEBUG_MODE2); in gen12_invalidate_state_cache()
1346 *cs++ = _MASKED_BIT_ENABLE(INSTRUCTION_STATE_CACHE_INVALIDATE); in gen12_invalidate_state_cache()
1347 return cs; in gen12_invalidate_state_cache()
1351 gen12_emit_indirect_ctx_rcs(const struct intel_context *ce, u32 *cs) in gen12_emit_indirect_ctx_rcs() argument
1353 cs = gen12_emit_timestamp_wa(ce, cs); in gen12_emit_indirect_ctx_rcs()
1354 cs = gen12_emit_cmd_buf_wa(ce, cs); in gen12_emit_indirect_ctx_rcs()
1355 cs = gen12_emit_restore_scratch(ce, cs); in gen12_emit_indirect_ctx_rcs()
1358 if (IS_DG2_G11(ce->engine->i915)) in gen12_emit_indirect_ctx_rcs()
1359 cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE, 0); in gen12_emit_indirect_ctx_rcs()
1361 cs = gen12_emit_aux_table_inv(ce->engine, cs); in gen12_emit_indirect_ctx_rcs()
1364 if (IS_GFX_GT_IP_RANGE(ce->engine->gt, IP_VER(12, 0), IP_VER(12, 10))) in gen12_emit_indirect_ctx_rcs()
1365 cs = gen12_invalidate_state_cache(cs); in gen12_emit_indirect_ctx_rcs()
1368 if (IS_GFX_GT_IP_STEP(ce->engine->gt, IP_VER(12, 70), STEP_A0, STEP_B0) || in gen12_emit_indirect_ctx_rcs()
1369 IS_GFX_GT_IP_STEP(ce->engine->gt, IP_VER(12, 71), STEP_A0, STEP_B0) || in gen12_emit_indirect_ctx_rcs()
1370 IS_DG2(ce->engine->i915)) in gen12_emit_indirect_ctx_rcs()
1371 cs = dg2_emit_draw_watermark_setting(cs); in gen12_emit_indirect_ctx_rcs()
1373 return cs; in gen12_emit_indirect_ctx_rcs()
1377 gen12_emit_indirect_ctx_xcs(const struct intel_context *ce, u32 *cs) in gen12_emit_indirect_ctx_xcs() argument
1379 cs = gen12_emit_timestamp_wa(ce, cs); in gen12_emit_indirect_ctx_xcs()
1380 cs = gen12_emit_restore_scratch(ce, cs); in gen12_emit_indirect_ctx_xcs()
1383 if (IS_DG2_G11(ce->engine->i915)) in gen12_emit_indirect_ctx_xcs()
1384 if (ce->engine->class == COMPUTE_CLASS) in gen12_emit_indirect_ctx_xcs()
1385 cs = gen8_emit_pipe_control(cs, in gen12_emit_indirect_ctx_xcs()
1389 return gen12_emit_aux_table_inv(ce->engine, cs); in gen12_emit_indirect_ctx_xcs()
1392 static u32 *xehp_emit_fastcolor_blt_wabb(const struct intel_context *ce, u32 *cs) in xehp_emit_fastcolor_blt_wabb() argument
1394 struct intel_gt *gt = ce->engine->gt; in xehp_emit_fastcolor_blt_wabb()
1395 int mocs = gt->mocs.uc_index << 1; in xehp_emit_fastcolor_blt_wabb()
1405 * BG0 -> 5100000E in xehp_emit_fastcolor_blt_wabb()
1406 * BG1 -> 0000003F (Dest pitch) in xehp_emit_fastcolor_blt_wabb()
1407 * BG2 -> 00000000 (X1, Y1) = (0, 0) in xehp_emit_fastcolor_blt_wabb()
1408 * BG3 -> 00040001 (X2, Y2) = (1, 4) in xehp_emit_fastcolor_blt_wabb()
1409 * BG4 -> scratch in xehp_emit_fastcolor_blt_wabb()
1410 * BG5 -> scratch in xehp_emit_fastcolor_blt_wabb()
1411 * BG6-12 -> 00000000 in xehp_emit_fastcolor_blt_wabb()
1412 * BG13 -> 20004004 (Surf. Width= 2,Surf. Height = 5 ) in xehp_emit_fastcolor_blt_wabb()
1413 * BG14 -> 00000010 (Qpitch = 4) in xehp_emit_fastcolor_blt_wabb()
1414 * BG15 -> 00000000 in xehp_emit_fastcolor_blt_wabb()
1416 *cs++ = XY_FAST_COLOR_BLT_CMD | (16 - 2); in xehp_emit_fastcolor_blt_wabb()
1417 *cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, mocs) | 0x3f; in xehp_emit_fastcolor_blt_wabb()
1418 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1419 *cs++ = 4 << 16 | 1; in xehp_emit_fastcolor_blt_wabb()
1420 *cs++ = lower_32_bits(i915_vma_offset(ce->vm->rsvd.vma)); in xehp_emit_fastcolor_blt_wabb()
1421 *cs++ = upper_32_bits(i915_vma_offset(ce->vm->rsvd.vma)); in xehp_emit_fastcolor_blt_wabb()
1422 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1423 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1424 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1425 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1426 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1427 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1428 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1429 *cs++ = 0x20004004; in xehp_emit_fastcolor_blt_wabb()
1430 *cs++ = 0x10; in xehp_emit_fastcolor_blt_wabb()
1431 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1433 return cs; in xehp_emit_fastcolor_blt_wabb()
1437 xehp_emit_per_ctx_bb(const struct intel_context *ce, u32 *cs) in xehp_emit_per_ctx_bb() argument
1440 if (NEEDS_FASTCOLOR_BLT_WABB(ce->engine)) in xehp_emit_per_ctx_bb()
1441 cs = xehp_emit_fastcolor_blt_wabb(ce, cs); in xehp_emit_per_ctx_bb()
1443 return cs; in xehp_emit_per_ctx_bb()
1453 u32 *cs; in setup_per_ctx_bb() local
1455 cs = emit(ce, start); in setup_per_ctx_bb()
1458 *cs++ = MI_BATCH_BUFFER_END; in setup_per_ctx_bb()
1460 GEM_BUG_ON(cs - start > I915_GTT_PAGE_SIZE / sizeof(*cs)); in setup_per_ctx_bb()
1461 lrc_setup_bb_per_ctx(ce->lrc_reg_state, engine, in setup_per_ctx_bb()
1471 u32 *cs; in setup_indirect_ctx_bb() local
1473 cs = emit(ce, start); in setup_indirect_ctx_bb()
1474 GEM_BUG_ON(cs - start > I915_GTT_PAGE_SIZE / sizeof(*cs)); in setup_indirect_ctx_bb()
1475 while ((unsigned long)cs % CACHELINE_BYTES) in setup_indirect_ctx_bb()
1476 *cs++ = MI_NOOP; in setup_indirect_ctx_bb()
1478 GEM_BUG_ON(cs - start > DG2_PREDICATE_RESULT_BB / sizeof(*start)); in setup_indirect_ctx_bb()
1481 lrc_setup_indirect_ctx(ce->lrc_reg_state, engine, in setup_indirect_ctx_bb()
1483 (cs - start) * sizeof(*cs)); in setup_indirect_ctx_bb()
1494 * bits 0-11: flags, GEN8_CTX_* (cached in ctx->desc_template)
1495 * bits 12-31: LRCA, GTT address of (the HWSP of) this context
1496 * bits 32-52: ctx ID, a globally unique tag (highest bit used by GuC)
1497 * bits 53-54: mbz, reserved for use by hardware
1498 * bits 55-63: group ID, currently unused and set to 0
1502 * bits 32-36: reserved
1503 * bits 37-47: SW context ID
1506 * bits 55-60: SW counter
1507 * bits 61-63: engine class
1511 * bits 32-37: virtual function number
1513 * bits 39-54: SW context ID
1514 * bits 55-57: reserved
1515 * bits 58-63: SW counter
1525 if (i915_vm_is_4lvl(ce->vm)) in lrc_descriptor()
1530 if (GRAPHICS_VER(ce->vm->i915) == 8) in lrc_descriptor()
1533 return i915_ggtt_offset(ce->state) | desc; in lrc_descriptor()
1540 struct intel_ring *ring = ce->ring; in lrc_update_regs()
1541 u32 *regs = ce->lrc_reg_state; in lrc_update_regs()
1544 GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->tail)); in lrc_update_regs()
1546 regs[CTX_RING_START] = i915_ggtt_offset(ring->vma); in lrc_update_regs()
1548 regs[CTX_RING_TAIL] = ring->tail; in lrc_update_regs()
1549 regs[CTX_RING_CTL] = RING_CTL_SIZE(ring->size) | RING_VALID; in lrc_update_regs()
1552 if (engine->class == RENDER_CLASS) { in lrc_update_regs()
1554 intel_sseu_make_rpcs(engine->gt, &ce->sseu); in lrc_update_regs()
1559 if (ce->wa_bb_page) { in lrc_update_regs()
1560 u32 *(*fn)(const struct intel_context *ce, u32 *cs); in lrc_update_regs()
1563 if (ce->engine->class == RENDER_CLASS) in lrc_update_regs()
1567 GEM_BUG_ON(engine->wa_ctx.indirect_ctx.size); in lrc_update_regs()
1578 set_offsets(ce->lrc_reg_state, reg_offsets(engine), engine, false); in lrc_update_offsets()
1585 const struct intel_ring *ring = ce->ring; in lrc_check_regs()
1586 u32 *regs = ce->lrc_reg_state; in lrc_check_regs()
1590 if (regs[CTX_RING_START] != i915_ggtt_offset(ring->vma)) { in lrc_check_regs()
1592 engine->name, in lrc_check_regs()
1594 i915_ggtt_offset(ring->vma)); in lrc_check_regs()
1595 regs[CTX_RING_START] = i915_ggtt_offset(ring->vma); in lrc_check_regs()
1600 (RING_CTL_SIZE(ring->size) | RING_VALID)) { in lrc_check_regs()
1602 engine->name, in lrc_check_regs()
1604 (u32)(RING_CTL_SIZE(ring->size) | RING_VALID)); in lrc_check_regs()
1605 regs[CTX_RING_CTL] = RING_CTL_SIZE(ring->size) | RING_VALID; in lrc_check_regs()
1610 if (x != -1 && regs[x + 1] & (regs[x + 1] >> 16) & STOP_RING) { in lrc_check_regs()
1612 engine->name, regs[x + 1]); in lrc_check_regs()
1630 * it for a short period and this batch in non-premptible. We can ofcourse
1643 *batch++ = intel_gt_scratch_offset(engine->gt, in gen8_emit_flush_coherentl3_wa()
1658 *batch++ = intel_gt_scratch_offset(engine->gt, in gen8_emit_flush_coherentl3_wa()
1686 if (IS_BROADWELL(engine->i915)) in gen8_init_indirectctx_bb()
1724 *batch++ = i915_mmio_reg_offset(lri->reg); in emit_lri()
1725 *batch++ = lri->value; in emit_lri()
1726 } while (lri++, --count); in emit_lri()
1773 if (HAS_POOLED_EU(engine->i915)) { in gen9_init_indirectctx_bb()
1777 * device type (2x6 or 3x6) and needs to be updated based in gen9_init_indirectctx_bb()
1778 * on which subslice is disabled especially for 2x6 in gen9_init_indirectctx_bb()
1812 obj = i915_gem_object_create_shmem(engine->i915, CTX_WA_BB_SIZE); in lrc_create_wa_ctx()
1816 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); in lrc_create_wa_ctx()
1822 engine->wa_ctx.vma = vma; in lrc_create_wa_ctx()
1832 i915_vma_unpin_and_release(&engine->wa_ctx.vma, 0); in lrc_fini_wa_ctx()
1839 struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx; in lrc_init_wa_ctx()
1841 &wa_ctx->indirect_ctx, &wa_ctx->per_ctx in lrc_init_wa_ctx()
1849 if (GRAPHICS_VER(engine->i915) >= 11 || in lrc_init_wa_ctx()
1850 !(engine->flags & I915_ENGINE_HAS_RCS_REG_STATE)) in lrc_init_wa_ctx()
1853 if (GRAPHICS_VER(engine->i915) == 9) { in lrc_init_wa_ctx()
1856 } else if (GRAPHICS_VER(engine->i915) == 8) { in lrc_init_wa_ctx()
1868 drm_err(&engine->i915->drm, in lrc_init_wa_ctx()
1874 if (!engine->wa_ctx.vma) in lrc_init_wa_ctx()
1879 err = i915_gem_object_lock(wa_ctx->vma->obj, &ww); in lrc_init_wa_ctx()
1881 err = i915_ggtt_pin(wa_ctx->vma, &ww, 0, PIN_HIGH); in lrc_init_wa_ctx()
1885 batch = i915_gem_object_pin_map(wa_ctx->vma->obj, I915_MAP_WB); in lrc_init_wa_ctx()
1898 wa_bb[i]->offset = batch_ptr - batch; in lrc_init_wa_ctx()
1899 if (GEM_DEBUG_WARN_ON(!IS_ALIGNED(wa_bb[i]->offset, in lrc_init_wa_ctx()
1901 err = -EINVAL; in lrc_init_wa_ctx()
1906 wa_bb[i]->size = batch_ptr - (batch + wa_bb[i]->offset); in lrc_init_wa_ctx()
1908 GEM_BUG_ON(batch_ptr - batch > CTX_WA_BB_SIZE); in lrc_init_wa_ctx()
1910 __i915_gem_object_flush_map(wa_ctx->vma->obj, 0, batch_ptr - batch); in lrc_init_wa_ctx()
1911 __i915_gem_object_release_map(wa_ctx->vma->obj); in lrc_init_wa_ctx()
1915 err = i915_inject_probe_error(engine->i915, -ENODEV); in lrc_init_wa_ctx()
1919 i915_vma_unpin(wa_ctx->vma); in lrc_init_wa_ctx()
1921 if (err == -EDEADLK) { in lrc_init_wa_ctx()
1929 i915_vma_put(engine->wa_ctx.vma); in lrc_init_wa_ctx()
1939 stats->runtime.num_underflow++; in st_runtime_underflow()
1940 stats->runtime.max_underflow = in st_runtime_underflow()
1941 max_t(u32, stats->runtime.max_underflow, -dt); in st_runtime_underflow()
1953 return READ_ONCE(ce->lrc_reg_state[CTX_TIMESTAMP]); in lrc_get_runtime()
1958 struct intel_context_stats *stats = &ce->stats; in lrc_update_runtime()
1962 old = stats->runtime.last; in lrc_update_runtime()
1963 stats->runtime.last = lrc_get_runtime(ce); in lrc_update_runtime()
1964 dt = stats->runtime.last - old; in lrc_update_runtime()
1970 old, stats->runtime.last, dt); in lrc_update_runtime()
1975 ewma_runtime_add(&stats->runtime.avg, dt); in lrc_update_runtime()
1976 stats->runtime.total += dt; in lrc_update_runtime()