Lines Matching +full:cs +full:- +full:1
1 // SPDX-License-Identifier: MIT
24 * The per-platform tables are u8-encoded in @data. Decode @data and set the
29 * [7]: create NOPs - number of NOPs are set in lower bits
60 const u32 base = engine->mmio_base; in set_offsets()
78 if (GRAPHICS_VER(engine->i915) >= 11) in set_offsets()
95 } while (--count); in set_offsets()
101 if (GRAPHICS_VER(engine->i915) >= 11) in set_offsets()
107 NOP(1),
142 NOP(1),
172 LRI(1, POSTED),
226 NOP(1),
258 NOP(1),
276 NOP(1),
292 NOP(1),
322 LRI(1, 0),
329 NOP(1),
359 LRI(1, 0),
413 NOP(1),
431 NOP(1),
443 LRI(1, POSTED),
447 LRI(1, 0),
454 NOP(1),
488 LRI(1, 0),
490 NOP(3 + 9 + 1),
544 NOP(1),
550 NOP(1),
568 NOP(1),
586 LRI(1, 0),
593 NOP(1),
611 NOP(1),
629 LRI(1, 0),
649 GEM_BUG_ON(GRAPHICS_VER(engine->i915) >= 12 && in reg_offsets()
652 if (engine->flags & I915_ENGINE_HAS_RCS_REG_STATE) { in reg_offsets()
653 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 70)) in reg_offsets()
655 else if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) in reg_offsets()
657 else if (GRAPHICS_VER(engine->i915) >= 12) in reg_offsets()
659 else if (GRAPHICS_VER(engine->i915) >= 11) in reg_offsets()
661 else if (GRAPHICS_VER(engine->i915) >= 9) in reg_offsets()
666 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) in reg_offsets()
668 else if (GRAPHICS_VER(engine->i915) >= 12) in reg_offsets()
670 else if (GRAPHICS_VER(engine->i915) >= 9) in reg_offsets()
679 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) in lrc_ring_mi_mode()
681 else if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_mi_mode()
683 else if (GRAPHICS_VER(engine->i915) >= 9) in lrc_ring_mi_mode()
685 else if (engine->class == RENDER_CLASS) in lrc_ring_mi_mode()
688 return -1; in lrc_ring_mi_mode()
693 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) in lrc_ring_bb_offset()
695 else if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_bb_offset()
697 else if (GRAPHICS_VER(engine->i915) >= 9) in lrc_ring_bb_offset()
699 else if (GRAPHICS_VER(engine->i915) >= 8 && in lrc_ring_bb_offset()
700 engine->class == RENDER_CLASS) in lrc_ring_bb_offset()
703 return -1; in lrc_ring_bb_offset()
708 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) in lrc_ring_gpr0()
710 else if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_gpr0()
712 else if (GRAPHICS_VER(engine->i915) >= 9) in lrc_ring_gpr0()
714 else if (engine->class == RENDER_CLASS) in lrc_ring_gpr0()
717 return -1; in lrc_ring_gpr0()
722 if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_wa_bb_per_ctx()
724 else if (GRAPHICS_VER(engine->i915) >= 9 || engine->class == RENDER_CLASS) in lrc_ring_wa_bb_per_ctx()
727 return -1; in lrc_ring_wa_bb_per_ctx()
755 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) in lrc_ring_cmd_buf_cctl()
761 else if (engine->class != RENDER_CLASS) in lrc_ring_cmd_buf_cctl()
762 return -1; in lrc_ring_cmd_buf_cctl()
763 else if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_cmd_buf_cctl()
765 else if (GRAPHICS_VER(engine->i915) >= 11) in lrc_ring_cmd_buf_cctl()
768 return -1; in lrc_ring_cmd_buf_cctl()
774 if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_indirect_offset_default()
776 else if (GRAPHICS_VER(engine->i915) >= 11) in lrc_ring_indirect_offset_default()
778 else if (GRAPHICS_VER(engine->i915) >= 9) in lrc_ring_indirect_offset_default()
780 else if (GRAPHICS_VER(engine->i915) >= 8) in lrc_ring_indirect_offset_default()
783 GEM_BUG_ON(GRAPHICS_VER(engine->i915) < 8); in lrc_ring_indirect_offset_default()
793 GEM_BUG_ON(lrc_ring_wa_bb_per_ctx(engine) == -1); in lrc_setup_bb_per_ctx()
794 regs[lrc_ring_wa_bb_per_ctx(engine) + 1] = in lrc_setup_bb_per_ctx()
808 GEM_BUG_ON(lrc_ring_indirect_ptr(engine) == -1); in lrc_setup_indirect_ctx()
809 regs[lrc_ring_indirect_ptr(engine) + 1] = in lrc_setup_indirect_ctx()
812 GEM_BUG_ON(lrc_ring_indirect_offset(engine) == -1); in lrc_setup_indirect_ctx()
813 regs[lrc_ring_indirect_offset(engine) + 1] = in lrc_setup_indirect_ctx()
823 * Wa_14019159160 - Case 2. in ctx_needs_runalone()
825 * the LRC run-alone bit or else the encryption/decryption will not happen. in ctx_needs_runalone()
826 * NOTE: Case 2 only applies to PXP use-case of said workaround. in ctx_needs_runalone()
828 if (GRAPHICS_VER_FULL(ce->engine->i915) >= IP_VER(12, 70) && in ctx_needs_runalone()
829 (ce->engine->class == COMPUTE_CLASS || ce->engine->class == RENDER_CLASS)) { in ctx_needs_runalone()
831 gem_ctx = rcu_dereference(ce->gem_context); in ctx_needs_runalone()
833 ctx_is_protected = gem_ctx->uses_protected_content; in ctx_needs_runalone()
852 if (GRAPHICS_VER(engine->i915) < 11) in init_common_regs()
855 /* Wa_14019159160 - Case 2.*/ in init_common_regs()
860 regs[CTX_TIMESTAMP] = ce->stats.runtime.last; in init_common_regs()
863 if (loc != -1) in init_common_regs()
864 regs[loc + 1] = 0; in init_common_regs()
870 const struct i915_ctx_workarounds * const wa_ctx = &engine->wa_ctx; in init_wa_bb_regs()
872 if (wa_ctx->per_ctx.size) { in init_wa_bb_regs()
873 const u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma); in init_wa_bb_regs()
875 GEM_BUG_ON(lrc_ring_wa_bb_per_ctx(engine) == -1); in init_wa_bb_regs()
876 regs[lrc_ring_wa_bb_per_ctx(engine) + 1] = in init_wa_bb_regs()
877 (ggtt_offset + wa_ctx->per_ctx.offset) | 0x01; in init_wa_bb_regs()
880 if (wa_ctx->indirect_ctx.size) { in init_wa_bb_regs()
882 i915_ggtt_offset(wa_ctx->vma) + in init_wa_bb_regs()
883 wa_ctx->indirect_ctx.offset, in init_wa_bb_regs()
884 wa_ctx->indirect_ctx.size); in init_wa_bb_regs()
890 if (i915_vm_is_4lvl(&ppgtt->vm)) { in init_ppgtt_regs()
899 ASSIGN_CTX_PDP(ppgtt, regs, 1); in init_ppgtt_regs()
907 return i915_vm_to_ggtt(vm)->alias; in vm_alias()
917 if (x != -1) { in __reset_stop_ring()
918 regs[x + 1] &= ~STOP_RING; in __reset_stop_ring()
919 regs[x + 1] |= STOP_RING << 16; in __reset_stop_ring()
945 init_ppgtt_regs(regs, vm_alias(ce->vm)); in __lrc_init_regs()
956 __lrc_init_regs(ce->lrc_reg_state, ce, engine, inhibit); in lrc_init_regs()
962 __reset_stop_ring(ce->lrc_reg_state, engine); in lrc_reset_regs()
971 vaddr += engine->context_size; in set_redzone()
982 vaddr += engine->context_size; in check_redzone()
985 drm_err_once(&engine->i915->drm, in check_redzone()
987 engine->name); in check_redzone()
992 return PAGE_SIZE * ce->wa_bb_page; in context_wa_bb_offset()
1005 GEM_BUG_ON(!ce->wa_bb_page); in context_wabb()
1007 ptr = ce->lrc_reg_state; in context_wabb()
1008 ptr -= LRC_STATE_OFFSET; /* back to start of context image */ in context_wabb()
1023 if (ce->default_state) { in lrc_init_state()
1024 shmem_read(ce->default_state, 0, state, engine->context_size); in lrc_init_state()
1025 __set_bit(CONTEXT_VALID_BIT, &ce->flags); in lrc_init_state()
1029 /* Clear the ppHWSP (inc. per-context counters) */ in lrc_init_state()
1033 if (ce->wa_bb_page) in lrc_init_state()
1045 return i915_ggtt_offset(ce->state) + context_wa_bb_offset(ce); in lrc_indirect_bb()
1048 static u32 *setup_predicate_disable_wa(const struct intel_context *ce, u32 *cs) in setup_predicate_disable_wa() argument
1051 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT | (4 - 2); in setup_predicate_disable_wa()
1052 *cs++ = lrc_indirect_bb(ce) + DG2_PREDICATE_RESULT_WA; in setup_predicate_disable_wa()
1053 *cs++ = 0; in setup_predicate_disable_wa()
1054 *cs++ = 0; /* No predication */ in setup_predicate_disable_wa()
1057 *cs++ = MI_BATCH_BUFFER_END | BIT(15); in setup_predicate_disable_wa()
1058 *cs++ = MI_SET_PREDICATE | MI_SET_PREDICATE_DISABLE; in setup_predicate_disable_wa()
1061 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT | (4 - 2); in setup_predicate_disable_wa()
1062 *cs++ = lrc_indirect_bb(ce) + DG2_PREDICATE_RESULT_WA; in setup_predicate_disable_wa()
1063 *cs++ = 0; in setup_predicate_disable_wa()
1064 *cs++ = 1; /* enable predication before the next BB */ in setup_predicate_disable_wa()
1066 *cs++ = MI_BATCH_BUFFER_END; in setup_predicate_disable_wa()
1067 GEM_BUG_ON(offset_in_page(cs) > DG2_PREDICATE_RESULT_WA); in setup_predicate_disable_wa()
1069 return cs; in setup_predicate_disable_wa()
1079 context_size = round_up(engine->context_size, I915_GTT_PAGE_SIZE); in __lrc_alloc_state()
1084 if (GRAPHICS_VER(engine->i915) >= 12) { in __lrc_alloc_state()
1085 ce->wa_bb_page = context_size / PAGE_SIZE; in __lrc_alloc_state()
1091 ce->parallel.guc.parent_page = context_size / PAGE_SIZE; in __lrc_alloc_state()
1095 obj = i915_gem_object_create_lmem(engine->i915, context_size, in __lrc_alloc_state()
1098 obj = i915_gem_object_create_shmem(engine->i915, context_size); in __lrc_alloc_state()
1107 if (intel_gt_needs_wa_22016122933(engine->gt)) in __lrc_alloc_state()
1111 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); in __lrc_alloc_state()
1123 struct intel_timeline *tl = fetch_and_zero(&ce->timeline); in pinned_timeline()
1134 GEM_BUG_ON(ce->state); in lrc_alloc()
1137 ce->default_state = engine->default_state; in lrc_alloc()
1143 ring = intel_engine_create_ring(engine, ce->ring_size); in lrc_alloc()
1149 if (!page_mask_bits(ce->timeline)) { in lrc_alloc()
1156 if (unlikely(ce->timeline)) in lrc_alloc()
1159 tl = intel_timeline_create(engine->gt); in lrc_alloc()
1165 ce->timeline = tl; in lrc_alloc()
1168 ce->ring = ring; in lrc_alloc()
1169 ce->state = vma; in lrc_alloc()
1184 intel_ring_reset(ce->ring, ce->ring->emit); in lrc_reset()
1187 lrc_init_regs(ce, ce->engine, true); in lrc_reset()
1188 ce->lrc.lrca = lrc_update_regs(ce, ce->engine, ce->ring->tail); in lrc_reset()
1197 GEM_BUG_ON(!ce->state); in lrc_pre_pin()
1198 GEM_BUG_ON(!i915_vma_is_pinned(ce->state)); in lrc_pre_pin()
1200 *vaddr = i915_gem_object_pin_map(ce->state->obj, in lrc_pre_pin()
1201 intel_gt_coherent_map_type(ce->engine->gt, in lrc_pre_pin()
1202 ce->state->obj, in lrc_pre_pin()
1214 ce->lrc_reg_state = vaddr + LRC_STATE_OFFSET; in lrc_pin()
1216 if (!__test_and_set_bit(CONTEXT_INIT_BIT, &ce->flags)) in lrc_pin()
1219 ce->lrc.lrca = lrc_update_regs(ce, engine, ce->ring->tail); in lrc_pin()
1225 if (unlikely(ce->parallel.last_rq)) { in lrc_unpin()
1226 i915_request_put(ce->parallel.last_rq); in lrc_unpin()
1227 ce->parallel.last_rq = NULL; in lrc_unpin()
1229 check_redzone((void *)ce->lrc_reg_state - LRC_STATE_OFFSET, in lrc_unpin()
1230 ce->engine); in lrc_unpin()
1235 i915_gem_object_unpin_map(ce->state->obj); in lrc_post_unpin()
1240 if (!ce->state) in lrc_fini()
1243 intel_ring_put(fetch_and_zero(&ce->ring)); in lrc_fini()
1244 i915_vma_put(fetch_and_zero(&ce->state)); in lrc_fini()
1251 GEM_BUG_ON(!i915_active_is_idle(&ce->active)); in lrc_destroy()
1261 gen12_emit_timestamp_wa(const struct intel_context *ce, u32 *cs) in gen12_emit_timestamp_wa() argument
1263 *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | in gen12_emit_timestamp_wa()
1266 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_timestamp_wa()
1267 *cs++ = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET + in gen12_emit_timestamp_wa()
1269 *cs++ = 0; in gen12_emit_timestamp_wa()
1271 *cs++ = MI_LOAD_REGISTER_REG | in gen12_emit_timestamp_wa()
1274 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_timestamp_wa()
1275 *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(0)); in gen12_emit_timestamp_wa()
1277 *cs++ = MI_LOAD_REGISTER_REG | in gen12_emit_timestamp_wa()
1280 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_timestamp_wa()
1281 *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(0)); in gen12_emit_timestamp_wa()
1283 return cs; in gen12_emit_timestamp_wa()
1287 gen12_emit_restore_scratch(const struct intel_context *ce, u32 *cs) in gen12_emit_restore_scratch() argument
1289 GEM_BUG_ON(lrc_ring_gpr0(ce->engine) == -1); in gen12_emit_restore_scratch()
1291 *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | in gen12_emit_restore_scratch()
1294 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_restore_scratch()
1295 *cs++ = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET + in gen12_emit_restore_scratch()
1296 (lrc_ring_gpr0(ce->engine) + 1) * sizeof(u32); in gen12_emit_restore_scratch()
1297 *cs++ = 0; in gen12_emit_restore_scratch()
1299 return cs; in gen12_emit_restore_scratch()
1303 gen12_emit_cmd_buf_wa(const struct intel_context *ce, u32 *cs) in gen12_emit_cmd_buf_wa() argument
1305 GEM_BUG_ON(lrc_ring_cmd_buf_cctl(ce->engine) == -1); in gen12_emit_cmd_buf_wa()
1307 *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | in gen12_emit_cmd_buf_wa()
1310 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_cmd_buf_wa()
1311 *cs++ = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET + in gen12_emit_cmd_buf_wa()
1312 (lrc_ring_cmd_buf_cctl(ce->engine) + 1) * sizeof(u32); in gen12_emit_cmd_buf_wa()
1313 *cs++ = 0; in gen12_emit_cmd_buf_wa()
1315 *cs++ = MI_LOAD_REGISTER_REG | in gen12_emit_cmd_buf_wa()
1318 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_cmd_buf_wa()
1319 *cs++ = i915_mmio_reg_offset(RING_CMD_BUF_CCTL(0)); in gen12_emit_cmd_buf_wa()
1321 return cs; in gen12_emit_cmd_buf_wa()
1332 dg2_emit_draw_watermark_setting(u32 *cs) in dg2_emit_draw_watermark_setting() argument
1334 *cs++ = MI_LOAD_REGISTER_IMM(1); in dg2_emit_draw_watermark_setting()
1335 *cs++ = i915_mmio_reg_offset(DRAW_WATERMARK); in dg2_emit_draw_watermark_setting()
1336 *cs++ = REG_FIELD_PREP(VERT_WM_VAL, 0x3FF); in dg2_emit_draw_watermark_setting()
1338 return cs; in dg2_emit_draw_watermark_setting()
1342 gen12_invalidate_state_cache(u32 *cs) in gen12_invalidate_state_cache() argument
1344 *cs++ = MI_LOAD_REGISTER_IMM(1); in gen12_invalidate_state_cache()
1345 *cs++ = i915_mmio_reg_offset(GEN12_CS_DEBUG_MODE2); in gen12_invalidate_state_cache()
1346 *cs++ = _MASKED_BIT_ENABLE(INSTRUCTION_STATE_CACHE_INVALIDATE); in gen12_invalidate_state_cache()
1347 return cs; in gen12_invalidate_state_cache()
1351 gen12_emit_indirect_ctx_rcs(const struct intel_context *ce, u32 *cs) in gen12_emit_indirect_ctx_rcs() argument
1353 cs = gen12_emit_timestamp_wa(ce, cs); in gen12_emit_indirect_ctx_rcs()
1354 cs = gen12_emit_cmd_buf_wa(ce, cs); in gen12_emit_indirect_ctx_rcs()
1355 cs = gen12_emit_restore_scratch(ce, cs); in gen12_emit_indirect_ctx_rcs()
1358 if (IS_DG2_G11(ce->engine->i915)) in gen12_emit_indirect_ctx_rcs()
1359 cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE, 0); in gen12_emit_indirect_ctx_rcs()
1361 cs = gen12_emit_aux_table_inv(ce->engine, cs); in gen12_emit_indirect_ctx_rcs()
1364 if (IS_GFX_GT_IP_RANGE(ce->engine->gt, IP_VER(12, 0), IP_VER(12, 10))) in gen12_emit_indirect_ctx_rcs()
1365 cs = gen12_invalidate_state_cache(cs); in gen12_emit_indirect_ctx_rcs()
1368 if (IS_GFX_GT_IP_STEP(ce->engine->gt, IP_VER(12, 70), STEP_A0, STEP_B0) || in gen12_emit_indirect_ctx_rcs()
1369 IS_GFX_GT_IP_STEP(ce->engine->gt, IP_VER(12, 71), STEP_A0, STEP_B0) || in gen12_emit_indirect_ctx_rcs()
1370 IS_DG2(ce->engine->i915)) in gen12_emit_indirect_ctx_rcs()
1371 cs = dg2_emit_draw_watermark_setting(cs); in gen12_emit_indirect_ctx_rcs()
1373 return cs; in gen12_emit_indirect_ctx_rcs()
1377 gen12_emit_indirect_ctx_xcs(const struct intel_context *ce, u32 *cs) in gen12_emit_indirect_ctx_xcs() argument
1379 cs = gen12_emit_timestamp_wa(ce, cs); in gen12_emit_indirect_ctx_xcs()
1380 cs = gen12_emit_restore_scratch(ce, cs); in gen12_emit_indirect_ctx_xcs()
1383 if (IS_DG2_G11(ce->engine->i915)) in gen12_emit_indirect_ctx_xcs()
1384 if (ce->engine->class == COMPUTE_CLASS) in gen12_emit_indirect_ctx_xcs()
1385 cs = gen8_emit_pipe_control(cs, in gen12_emit_indirect_ctx_xcs()
1389 return gen12_emit_aux_table_inv(ce->engine, cs); in gen12_emit_indirect_ctx_xcs()
1392 static u32 *xehp_emit_fastcolor_blt_wabb(const struct intel_context *ce, u32 *cs) in xehp_emit_fastcolor_blt_wabb() argument
1394 struct intel_gt *gt = ce->engine->gt; in xehp_emit_fastcolor_blt_wabb()
1395 int mocs = gt->mocs.uc_index << 1; in xehp_emit_fastcolor_blt_wabb()
1405 * BG0 -> 5100000E in xehp_emit_fastcolor_blt_wabb()
1406 * BG1 -> 0000003F (Dest pitch) in xehp_emit_fastcolor_blt_wabb()
1407 * BG2 -> 00000000 (X1, Y1) = (0, 0) in xehp_emit_fastcolor_blt_wabb()
1408 * BG3 -> 00040001 (X2, Y2) = (1, 4) in xehp_emit_fastcolor_blt_wabb()
1409 * BG4 -> scratch in xehp_emit_fastcolor_blt_wabb()
1410 * BG5 -> scratch in xehp_emit_fastcolor_blt_wabb()
1411 * BG6-12 -> 00000000 in xehp_emit_fastcolor_blt_wabb()
1412 * BG13 -> 20004004 (Surf. Width= 2,Surf. Height = 5 ) in xehp_emit_fastcolor_blt_wabb()
1413 * BG14 -> 00000010 (Qpitch = 4) in xehp_emit_fastcolor_blt_wabb()
1414 * BG15 -> 00000000 in xehp_emit_fastcolor_blt_wabb()
1416 *cs++ = XY_FAST_COLOR_BLT_CMD | (16 - 2); in xehp_emit_fastcolor_blt_wabb()
1417 *cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, mocs) | 0x3f; in xehp_emit_fastcolor_blt_wabb()
1418 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1419 *cs++ = 4 << 16 | 1; in xehp_emit_fastcolor_blt_wabb()
1420 *cs++ = lower_32_bits(i915_vma_offset(ce->vm->rsvd.vma)); in xehp_emit_fastcolor_blt_wabb()
1421 *cs++ = upper_32_bits(i915_vma_offset(ce->vm->rsvd.vma)); in xehp_emit_fastcolor_blt_wabb()
1422 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1423 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1424 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1425 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1426 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1427 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1428 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1429 *cs++ = 0x20004004; in xehp_emit_fastcolor_blt_wabb()
1430 *cs++ = 0x10; in xehp_emit_fastcolor_blt_wabb()
1431 *cs++ = 0; in xehp_emit_fastcolor_blt_wabb()
1433 return cs; in xehp_emit_fastcolor_blt_wabb()
1437 xehp_emit_per_ctx_bb(const struct intel_context *ce, u32 *cs) in xehp_emit_per_ctx_bb() argument
1440 if (NEEDS_FASTCOLOR_BLT_WABB(ce->engine)) in xehp_emit_per_ctx_bb()
1441 cs = xehp_emit_fastcolor_blt_wabb(ce, cs); in xehp_emit_per_ctx_bb()
1443 return cs; in xehp_emit_per_ctx_bb()
1453 u32 *cs; in setup_per_ctx_bb() local
1455 cs = emit(ce, start); in setup_per_ctx_bb()
1458 *cs++ = MI_BATCH_BUFFER_END; in setup_per_ctx_bb()
1460 GEM_BUG_ON(cs - start > I915_GTT_PAGE_SIZE / sizeof(*cs)); in setup_per_ctx_bb()
1461 lrc_setup_bb_per_ctx(ce->lrc_reg_state, engine, in setup_per_ctx_bb()
1471 u32 *cs; in setup_indirect_ctx_bb() local
1473 cs = emit(ce, start); in setup_indirect_ctx_bb()
1474 GEM_BUG_ON(cs - start > I915_GTT_PAGE_SIZE / sizeof(*cs)); in setup_indirect_ctx_bb()
1475 while ((unsigned long)cs % CACHELINE_BYTES) in setup_indirect_ctx_bb()
1476 *cs++ = MI_NOOP; in setup_indirect_ctx_bb()
1478 GEM_BUG_ON(cs - start > DG2_PREDICATE_RESULT_BB / sizeof(*start)); in setup_indirect_ctx_bb()
1481 lrc_setup_indirect_ctx(ce->lrc_reg_state, engine, in setup_indirect_ctx_bb()
1483 (cs - start) * sizeof(*cs)); in setup_indirect_ctx_bb()
1494 * bits 0-11: flags, GEN8_CTX_* (cached in ctx->desc_template)
1495 * bits 12-31: LRCA, GTT address of (the HWSP of) this context
1496 * bits 32-52: ctx ID, a globally unique tag (highest bit used by GuC)
1497 * bits 53-54: mbz, reserved for use by hardware
1498 * bits 55-63: group ID, currently unused and set to 0
1502 * bits 32-36: reserved
1503 * bits 37-47: SW context ID
1506 * bits 55-60: SW counter
1507 * bits 61-63: engine class
1511 * bits 32-37: virtual function number
1513 * bits 39-54: SW context ID
1514 * bits 55-57: reserved
1515 * bits 58-63: SW counter
1525 if (i915_vm_is_4lvl(ce->vm)) in lrc_descriptor()
1530 if (GRAPHICS_VER(ce->vm->i915) == 8) in lrc_descriptor()
1533 return i915_ggtt_offset(ce->state) | desc; in lrc_descriptor()
1540 struct intel_ring *ring = ce->ring; in lrc_update_regs()
1541 u32 *regs = ce->lrc_reg_state; in lrc_update_regs()
1544 GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->tail)); in lrc_update_regs()
1546 regs[CTX_RING_START] = i915_ggtt_offset(ring->vma); in lrc_update_regs()
1548 regs[CTX_RING_TAIL] = ring->tail; in lrc_update_regs()
1549 regs[CTX_RING_CTL] = RING_CTL_SIZE(ring->size) | RING_VALID; in lrc_update_regs()
1552 if (engine->class == RENDER_CLASS) { in lrc_update_regs()
1554 intel_sseu_make_rpcs(engine->gt, &ce->sseu); in lrc_update_regs()
1559 if (ce->wa_bb_page) { in lrc_update_regs()
1560 u32 *(*fn)(const struct intel_context *ce, u32 *cs); in lrc_update_regs()
1563 if (ce->engine->class == RENDER_CLASS) in lrc_update_regs()
1567 GEM_BUG_ON(engine->wa_ctx.indirect_ctx.size); in lrc_update_regs()
1578 set_offsets(ce->lrc_reg_state, reg_offsets(engine), engine, false); in lrc_update_offsets()
1585 const struct intel_ring *ring = ce->ring; in lrc_check_regs()
1586 u32 *regs = ce->lrc_reg_state; in lrc_check_regs()
1590 if (regs[CTX_RING_START] != i915_ggtt_offset(ring->vma)) { in lrc_check_regs()
1592 engine->name, in lrc_check_regs()
1594 i915_ggtt_offset(ring->vma)); in lrc_check_regs()
1595 regs[CTX_RING_START] = i915_ggtt_offset(ring->vma); in lrc_check_regs()
1600 (RING_CTL_SIZE(ring->size) | RING_VALID)) { in lrc_check_regs()
1602 engine->name, in lrc_check_regs()
1604 (u32)(RING_CTL_SIZE(ring->size) | RING_VALID)); in lrc_check_regs()
1605 regs[CTX_RING_CTL] = RING_CTL_SIZE(ring->size) | RING_VALID; in lrc_check_regs()
1610 if (x != -1 && regs[x + 1] & (regs[x + 1] >> 16) & STOP_RING) { in lrc_check_regs()
1612 engine->name, regs[x + 1]); in lrc_check_regs()
1613 regs[x + 1] &= ~STOP_RING; in lrc_check_regs()
1614 regs[x + 1] |= STOP_RING << 16; in lrc_check_regs()
1630 * it for a short period and this batch in non-premptible. We can ofcourse
1643 *batch++ = intel_gt_scratch_offset(engine->gt, in gen8_emit_flush_coherentl3_wa()
1647 *batch++ = MI_LOAD_REGISTER_IMM(1); in gen8_emit_flush_coherentl3_wa()
1658 *batch++ = intel_gt_scratch_offset(engine->gt, in gen8_emit_flush_coherentl3_wa()
1686 if (IS_BROADWELL(engine->i915)) in gen8_init_indirectctx_bb()
1724 *batch++ = i915_mmio_reg_offset(lri->reg); in emit_lri()
1725 *batch++ = lri->value; in emit_lri()
1726 } while (lri++, --count); in emit_lri()
1773 if (HAS_POOLED_EU(engine->i915)) { in gen9_init_indirectctx_bb()
1812 obj = i915_gem_object_create_shmem(engine->i915, CTX_WA_BB_SIZE); in lrc_create_wa_ctx()
1816 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); in lrc_create_wa_ctx()
1822 engine->wa_ctx.vma = vma; in lrc_create_wa_ctx()
1832 i915_vma_unpin_and_release(&engine->wa_ctx.vma, 0); in lrc_fini_wa_ctx()
1839 struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx; in lrc_init_wa_ctx()
1841 &wa_ctx->indirect_ctx, &wa_ctx->per_ctx in lrc_init_wa_ctx()
1849 if (GRAPHICS_VER(engine->i915) >= 11 || in lrc_init_wa_ctx()
1850 !(engine->flags & I915_ENGINE_HAS_RCS_REG_STATE)) in lrc_init_wa_ctx()
1853 if (GRAPHICS_VER(engine->i915) == 9) { in lrc_init_wa_ctx()
1855 wa_bb_fn[1] = NULL; in lrc_init_wa_ctx()
1856 } else if (GRAPHICS_VER(engine->i915) == 8) { in lrc_init_wa_ctx()
1858 wa_bb_fn[1] = NULL; in lrc_init_wa_ctx()
1868 drm_err(&engine->i915->drm, in lrc_init_wa_ctx()
1874 if (!engine->wa_ctx.vma) in lrc_init_wa_ctx()
1879 err = i915_gem_object_lock(wa_ctx->vma->obj, &ww); in lrc_init_wa_ctx()
1881 err = i915_ggtt_pin(wa_ctx->vma, &ww, 0, PIN_HIGH); in lrc_init_wa_ctx()
1885 batch = i915_gem_object_pin_map(wa_ctx->vma->obj, I915_MAP_WB); in lrc_init_wa_ctx()
1898 wa_bb[i]->offset = batch_ptr - batch; in lrc_init_wa_ctx()
1899 if (GEM_DEBUG_WARN_ON(!IS_ALIGNED(wa_bb[i]->offset, in lrc_init_wa_ctx()
1901 err = -EINVAL; in lrc_init_wa_ctx()
1906 wa_bb[i]->size = batch_ptr - (batch + wa_bb[i]->offset); in lrc_init_wa_ctx()
1908 GEM_BUG_ON(batch_ptr - batch > CTX_WA_BB_SIZE); in lrc_init_wa_ctx()
1910 __i915_gem_object_flush_map(wa_ctx->vma->obj, 0, batch_ptr - batch); in lrc_init_wa_ctx()
1911 __i915_gem_object_release_map(wa_ctx->vma->obj); in lrc_init_wa_ctx()
1915 err = i915_inject_probe_error(engine->i915, -ENODEV); in lrc_init_wa_ctx()
1919 i915_vma_unpin(wa_ctx->vma); in lrc_init_wa_ctx()
1921 if (err == -EDEADLK) { in lrc_init_wa_ctx()
1929 i915_vma_put(engine->wa_ctx.vma); in lrc_init_wa_ctx()
1939 stats->runtime.num_underflow++; in st_runtime_underflow()
1940 stats->runtime.max_underflow = in st_runtime_underflow()
1941 max_t(u32, stats->runtime.max_underflow, -dt); in st_runtime_underflow()
1953 return READ_ONCE(ce->lrc_reg_state[CTX_TIMESTAMP]); in lrc_get_runtime()
1958 struct intel_context_stats *stats = &ce->stats; in lrc_update_runtime()
1962 old = stats->runtime.last; in lrc_update_runtime()
1963 stats->runtime.last = lrc_get_runtime(ce); in lrc_update_runtime()
1964 dt = stats->runtime.last - old; in lrc_update_runtime()
1970 old, stats->runtime.last, dt); in lrc_update_runtime()
1975 ewma_runtime_add(&stats->runtime.avg, dt); in lrc_update_runtime()
1976 stats->runtime.total += dt; in lrc_update_runtime()