Lines Matching +full:vm +full:- +full:map
1 // SPDX-License-Identifier: MIT
45 return gt_to_xe(lrc->fence_ctx.gt); in lrc_to_xe()
85 * The per-platform tables are u8-encoded in @data. Decode @data and set the
90 * [7]: create NOPs - number of NOPs are set in lower bits
120 const u32 base = hwe->mmio_base; in set_offsets()
141 xe_gt_assert(hwe->gt, count); in set_offsets()
154 } while (--count); in set_offsets()
576 if (xe_gt_has_indirect_ring_state(hwe->gt)) in set_context_control()
585 struct xe_memirq *memirq = >_to_tile(hwe->gt)->memirq; in set_memory_based_intr()
586 struct xe_device *xe = gt_to_xe(hwe->gt); in set_memory_based_intr()
613 struct xe_device *xe = gt_to_xe(hwe->gt); in lrc_ring_mi_mode()
632 return lrc->flags & XE_LRC_FLAG_INDIRECT_RING_STATE; in xe_lrc_has_indirect_ring_state()
642 return lrc->ring.size; in xe_lrc_pphwsp_offset()
675 /* The seqno is stored in the driver-defined portion of PPHWSP */ in __xe_lrc_seqno_offset()
681 /* The start seqno is stored in the driver-defined portion of PPHWSP */ in __xe_lrc_start_seqno_offset()
687 /* The start seqno is stored in the driver-defined portion of PPHWSP */ in __xe_lrc_ctx_job_timestamp_offset()
693 /* The parallel is stored in the driver-defined portion of PPHWSP */ in __xe_lrc_parallel_offset()
705 return lrc->size - LRC_INDIRECT_RING_STATE_SIZE; in __xe_lrc_indirect_ring_offset()
711 struct iosys_map map = lrc->bo->vmap; \
713 xe_assert(lrc_to_xe(lrc), !iosys_map_is_null(&map)); \
714 iosys_map_incr(&map, __xe_lrc_##elem##_offset(lrc)); \
715 return map; \
719 return xe_bo_ggtt_addr(lrc->bo) + __xe_lrc_##elem##_offset(lrc); \
735 * xe_lrc_ctx_timestamp_ggtt_addr() - Get ctx timestamp GGTT address in DECL_MAP_ADDR_HELPERS()
746 * xe_lrc_ctx_timestamp() - Read ctx timestamp value
754 struct iosys_map map; in xe_lrc_ctx_timestamp() local
756 map = __xe_lrc_ctx_timestamp_map(lrc); in xe_lrc_ctx_timestamp()
757 return xe_map_read32(xe, &map); in xe_lrc_ctx_timestamp()
761 * xe_lrc_ctx_job_timestamp_ggtt_addr() - Get ctx job timestamp GGTT address
772 * xe_lrc_ctx_job_timestamp() - Read ctx job timestamp value
780 struct iosys_map map; in xe_lrc_ctx_job_timestamp() local
782 map = __xe_lrc_ctx_job_timestamp_map(lrc); in xe_lrc_ctx_job_timestamp()
783 return xe_map_read32(xe, &map); in xe_lrc_ctx_job_timestamp()
802 struct iosys_map map; in xe_lrc_read_indirect_ctx_reg() local
804 map = __xe_lrc_indirect_ring_map(lrc); in xe_lrc_read_indirect_ctx_reg()
805 iosys_map_incr(&map, reg_nr * sizeof(u32)); in xe_lrc_read_indirect_ctx_reg()
806 return xe_map_read32(xe, &map); in xe_lrc_read_indirect_ctx_reg()
813 struct iosys_map map; in xe_lrc_write_indirect_ctx_reg() local
815 map = __xe_lrc_indirect_ring_map(lrc); in xe_lrc_write_indirect_ctx_reg()
816 iosys_map_incr(&map, reg_nr * sizeof(u32)); in xe_lrc_write_indirect_ctx_reg()
817 xe_map_write32(xe, &map, val); in xe_lrc_write_indirect_ctx_reg()
823 struct iosys_map map; in xe_lrc_read_ctx_reg() local
825 map = __xe_lrc_regs_map(lrc); in xe_lrc_read_ctx_reg()
826 iosys_map_incr(&map, reg_nr * sizeof(u32)); in xe_lrc_read_ctx_reg()
827 return xe_map_read32(xe, &map); in xe_lrc_read_ctx_reg()
833 struct iosys_map map; in xe_lrc_write_ctx_reg() local
835 map = __xe_lrc_regs_map(lrc); in xe_lrc_write_ctx_reg()
836 iosys_map_incr(&map, reg_nr * sizeof(u32)); in xe_lrc_write_ctx_reg()
837 xe_map_write32(xe, &map, val); in xe_lrc_write_ctx_reg()
842 struct xe_gt *gt = hwe->gt; in empty_lrc_data()
846 data = kzalloc(xe_gt_lrc_size(gt, hwe->class), GFP_KERNEL); in empty_lrc_data()
850 /* 1st page: Per-Process of HW status Page */ in empty_lrc_data()
852 set_offsets(regs, reg_offsets(gt_to_xe(gt), hwe->class), hwe); in empty_lrc_data()
857 regs = data + xe_gt_lrc_size(gt, hwe->class) - in empty_lrc_data()
865 static void xe_lrc_set_ppgtt(struct xe_lrc *lrc, struct xe_vm *vm) in xe_lrc_set_ppgtt() argument
867 u64 desc = xe_vm_pdp4_descriptor(vm, lrc->tile); in xe_lrc_set_ppgtt()
875 xe_hw_fence_ctx_finish(&lrc->fence_ctx); in xe_lrc_finish()
876 xe_bo_lock(lrc->bo, false); in xe_lrc_finish()
877 xe_bo_unpin(lrc->bo); in xe_lrc_finish()
878 xe_bo_unlock(lrc->bo); in xe_lrc_finish()
879 xe_bo_put(lrc->bo); in xe_lrc_finish()
886 struct xe_vm *vm, u32 ring_size, u16 msix_vec) in xe_lrc_init() argument
888 struct xe_gt *gt = hwe->gt; in xe_lrc_init()
891 struct iosys_map map; in xe_lrc_init() local
897 kref_init(&lrc->refcount); in xe_lrc_init()
898 lrc->flags = 0; in xe_lrc_init()
899 lrc_size = ring_size + xe_gt_lrc_size(gt, hwe->class); in xe_lrc_init()
901 lrc->flags |= XE_LRC_FLAG_INDIRECT_RING_STATE; in xe_lrc_init()
904 * FIXME: Perma-pinning LRC as we don't yet support moving GGTT address in xe_lrc_init()
905 * via VM bind calls. in xe_lrc_init()
907 lrc->bo = xe_bo_create_pin_map(xe, tile, vm, lrc_size, in xe_lrc_init()
912 if (IS_ERR(lrc->bo)) in xe_lrc_init()
913 return PTR_ERR(lrc->bo); in xe_lrc_init()
915 lrc->size = lrc_size; in xe_lrc_init()
916 lrc->tile = gt_to_tile(hwe->gt); in xe_lrc_init()
917 lrc->ring.size = ring_size; in xe_lrc_init()
918 lrc->ring.tail = 0; in xe_lrc_init()
919 lrc->ctx_timestamp = 0; in xe_lrc_init()
921 xe_hw_fence_ctx_init(&lrc->fence_ctx, hwe->gt, in xe_lrc_init()
922 hwe->fence_irq, hwe->name); in xe_lrc_init()
924 if (!gt->default_lrc[hwe->class]) { in xe_lrc_init()
927 err = -ENOMEM; in xe_lrc_init()
933 * Init Per-Process of HW status Page, LRC / context state to known in xe_lrc_init()
936 map = __xe_lrc_pphwsp_map(lrc); in xe_lrc_init()
938 xe_map_memset(xe, &map, 0, 0, LRC_PPHWSP_SIZE); /* PPHWSP */ in xe_lrc_init()
939 xe_map_memcpy_to(xe, &map, LRC_PPHWSP_SIZE, in xe_lrc_init()
940 gt->default_lrc[hwe->class] + LRC_PPHWSP_SIZE, in xe_lrc_init()
941 xe_gt_lrc_size(gt, hwe->class) - LRC_PPHWSP_SIZE); in xe_lrc_init()
943 xe_map_memcpy_to(xe, &map, 0, init_data, in xe_lrc_init()
944 xe_gt_lrc_size(gt, hwe->class)); in xe_lrc_init()
948 if (vm) { in xe_lrc_init()
949 xe_lrc_set_ppgtt(lrc, vm); in xe_lrc_init()
951 if (vm->xef) in xe_lrc_init()
952 xe_drm_client_add_bo(vm->xef->client, lrc->bo); in xe_lrc_init()
957 xe_memirq_status_ptr(&tile->memirq, hwe)); in xe_lrc_init()
959 xe_memirq_source_ptr(&tile->memirq, hwe)); in xe_lrc_init()
971 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_TAIL, lrc->ring.tail); in xe_lrc_init()
973 RING_CTL_SIZE(lrc->ring.size) | RING_VALID); in xe_lrc_init()
977 xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, lrc->ring.tail); in xe_lrc_init()
979 RING_CTL_SIZE(lrc->ring.size) | RING_VALID); in xe_lrc_init()
984 if (xe->info.has_asid && vm) in xe_lrc_init()
985 xe_lrc_write_ctx_reg(lrc, PVC_CTX_ASID, vm->usm.asid); in xe_lrc_init()
987 lrc->desc = LRC_VALID; in xe_lrc_init()
988 lrc->desc |= FIELD_PREP(LRC_ADDRESSING_MODE, LRC_LEGACY_64B_CONTEXT); in xe_lrc_init()
994 if (vm) in xe_lrc_init()
995 lrc->desc |= LRC_PRIVILEGE; in xe_lrc_init()
998 lrc->desc |= FIELD_PREP(LRC_ENGINE_INSTANCE, hwe->instance); in xe_lrc_init()
999 lrc->desc |= FIELD_PREP(LRC_ENGINE_CLASS, hwe->class); in xe_lrc_init()
1005 map = __xe_lrc_seqno_map(lrc); in xe_lrc_init()
1006 xe_map_write32(lrc_to_xe(lrc), &map, lrc->fence_ctx.next_seqno - 1); in xe_lrc_init()
1008 map = __xe_lrc_start_seqno_map(lrc); in xe_lrc_init()
1009 xe_map_write32(lrc_to_xe(lrc), &map, lrc->fence_ctx.next_seqno - 1); in xe_lrc_init()
1019 * xe_lrc_create - Create a LRC
1021 * @vm: The VM (address space)
1023 * @msix_vec: MSI-X interrupt vector (for platforms that support it)
1030 struct xe_lrc *xe_lrc_create(struct xe_hw_engine *hwe, struct xe_vm *vm, in xe_lrc_create() argument
1038 return ERR_PTR(-ENOMEM); in xe_lrc_create()
1040 err = xe_lrc_init(lrc, hwe, vm, ring_size, msix_vec); in xe_lrc_create()
1050 * xe_lrc_destroy - Destroy the LRC
1107 const u32 tail = lrc->ring.tail; in xe_lrc_ring_space()
1108 const u32 size = lrc->ring.size; in xe_lrc_ring_space()
1110 return ((head - tail - 1) & (size - 1)) + 1; in xe_lrc_ring_space()
1118 iosys_map_incr(&ring, lrc->ring.tail); in __xe_lrc_write_ring()
1120 lrc->ring.tail = (lrc->ring.tail + size) & (lrc->ring.size - 1); in __xe_lrc_write_ring()
1135 xe_assert(xe, lrc->ring.tail < lrc->ring.size); in xe_lrc_write_ring()
1136 rhs = lrc->ring.size - lrc->ring.tail; in xe_lrc_write_ring()
1139 __xe_lrc_write_ring(lrc, ring, data + rhs, size - rhs); in xe_lrc_write_ring()
1153 return lrc->desc | xe_lrc_ggtt_addr(lrc); in xe_lrc_descriptor()
1162 * xe_lrc_alloc_seqno_fence() - Allocate an lrc seqno fence.
1175 * xe_lrc_free_seqno_fence() - Free an lrc seqno fence.
1187 * xe_lrc_init_seqno_fence() - Initialize an lrc seqno fence.
1191 * Initializes a pre-allocated lrc seqno fence.
1193 * dma-fence refcounting.
1197 xe_hw_fence_init(fence, &lrc->fence_ctx, __xe_lrc_seqno_map(lrc)); in xe_lrc_init_seqno_fence()
1202 struct iosys_map map = __xe_lrc_seqno_map(lrc); in xe_lrc_seqno() local
1204 return xe_map_read32(lrc_to_xe(lrc), &map); in xe_lrc_seqno()
1209 struct iosys_map map = __xe_lrc_start_seqno_map(lrc); in xe_lrc_start_seqno() local
1211 return xe_map_read32(lrc_to_xe(lrc), &map); in xe_lrc_start_seqno()
1236 /* 3DSTATE_SO_DECL_LIST has a 9-bit dword length rather than 8 */ in instr_dw()
1284 inst_header, (numdw - 1) / 2); in dump_mi_command()
1286 drm_printf(p, " - %#6x = %#010x\n", dw[i], dw[i + 1]); in dump_mi_command()
1295 drm_printf(p, " - %#6x = %#010llx\n", in dump_mi_command()
1298 drm_printf(p, " - %*ph (%s)\n", in dump_mi_command()
1299 (int)sizeof(u32) * (numdw - 1), dw + 1, in dump_mi_command()
1325 * Make sure we haven't mis-parsed a number of dwords that exceeds the in dump_gfxpipe_command()
1474 * Make sure we haven't mis-parsed a number of dwords that exceeds the in dump_gfx_state_command()
1497 if (!gt->default_lrc[hwe_class]) { in xe_lrc_dump_default()
1503 * Skip the beginning of the LRC since it contains the per-process in xe_lrc_dump_default()
1506 dw = gt->default_lrc[hwe_class] + LRC_PPHWSP_SIZE; in xe_lrc_dump_default()
1507 remaining_dw = (xe_gt_lrc_size(gt, hwe_class) - LRC_PPHWSP_SIZE) / 4; in xe_lrc_dump_default()
1524 remaining_dw -= num_dw; in xe_lrc_dump_default()
1588 struct xe_gt *gt = q->hwe->gt; in xe_lrc_emit_hwe_state_instructions()
1613 if (XE_WA(gt, 14019789679) && q->hwe->class == XE_ENGINE_CLASS_RENDER) { in xe_lrc_emit_hwe_state_instructions()
1619 xe_gt_dbg(gt, "No non-register state to emit on graphics ver %d.%02d\n", in xe_lrc_emit_hwe_state_instructions()
1635 * except that 3DSTATE_DRAWING_RECTANGLE (non-pipelined) has in xe_lrc_emit_hwe_state_instructions()
1644 bb->cs[bb->len] = instr; in xe_lrc_emit_hwe_state_instructions()
1646 bb->cs[bb->len] |= (num_dw - 2); in xe_lrc_emit_hwe_state_instructions()
1648 bb->len += num_dw; in xe_lrc_emit_hwe_state_instructions()
1659 if (lrc->bo->vm) in xe_lrc_snapshot_capture()
1660 xe_vm_get(lrc->bo->vm); in xe_lrc_snapshot_capture()
1662 snapshot->context_desc = xe_lrc_ggtt_addr(lrc); in xe_lrc_snapshot_capture()
1663 snapshot->ring_addr = __xe_lrc_ring_ggtt_addr(lrc); in xe_lrc_snapshot_capture()
1664 snapshot->indirect_context_desc = xe_lrc_indirect_ring_ggtt_addr(lrc); in xe_lrc_snapshot_capture()
1665 snapshot->head = xe_lrc_ring_head(lrc); in xe_lrc_snapshot_capture()
1666 snapshot->tail.internal = lrc->ring.tail; in xe_lrc_snapshot_capture()
1667 snapshot->tail.memory = xe_lrc_ring_tail(lrc); in xe_lrc_snapshot_capture()
1668 snapshot->start = xe_lrc_ring_start(lrc); in xe_lrc_snapshot_capture()
1669 snapshot->start_seqno = xe_lrc_start_seqno(lrc); in xe_lrc_snapshot_capture()
1670 snapshot->seqno = xe_lrc_seqno(lrc); in xe_lrc_snapshot_capture()
1671 snapshot->lrc_bo = xe_bo_get(lrc->bo); in xe_lrc_snapshot_capture()
1672 snapshot->lrc_offset = xe_lrc_pphwsp_offset(lrc); in xe_lrc_snapshot_capture()
1673 snapshot->lrc_size = lrc->bo->size - snapshot->lrc_offset; in xe_lrc_snapshot_capture()
1674 snapshot->lrc_snapshot = NULL; in xe_lrc_snapshot_capture()
1675 snapshot->ctx_timestamp = xe_lrc_ctx_timestamp(lrc); in xe_lrc_snapshot_capture()
1676 snapshot->ctx_job_timestamp = xe_lrc_ctx_job_timestamp(lrc); in xe_lrc_snapshot_capture()
1683 struct xe_vm *vm; in xe_lrc_snapshot_capture_delayed() local
1689 bo = snapshot->lrc_bo; in xe_lrc_snapshot_capture_delayed()
1690 vm = bo->vm; in xe_lrc_snapshot_capture_delayed()
1691 snapshot->lrc_bo = NULL; in xe_lrc_snapshot_capture_delayed()
1693 snapshot->lrc_snapshot = kvmalloc(snapshot->lrc_size, GFP_KERNEL); in xe_lrc_snapshot_capture_delayed()
1694 if (!snapshot->lrc_snapshot) in xe_lrc_snapshot_capture_delayed()
1698 if (!ttm_bo_vmap(&bo->ttm, &src)) { in xe_lrc_snapshot_capture_delayed()
1700 snapshot->lrc_snapshot, &src, snapshot->lrc_offset, in xe_lrc_snapshot_capture_delayed()
1701 snapshot->lrc_size); in xe_lrc_snapshot_capture_delayed()
1702 ttm_bo_vunmap(&bo->ttm, &src); in xe_lrc_snapshot_capture_delayed()
1704 kvfree(snapshot->lrc_snapshot); in xe_lrc_snapshot_capture_delayed()
1705 snapshot->lrc_snapshot = NULL; in xe_lrc_snapshot_capture_delayed()
1710 if (vm) in xe_lrc_snapshot_capture_delayed()
1711 xe_vm_put(vm); in xe_lrc_snapshot_capture_delayed()
1721 drm_printf(p, "\tHW Context Desc: 0x%08x\n", snapshot->context_desc); in xe_lrc_snapshot_print()
1723 snapshot->ring_addr); in xe_lrc_snapshot_print()
1725 snapshot->indirect_context_desc); in xe_lrc_snapshot_print()
1726 drm_printf(p, "\tLRC Head: (memory) %u\n", snapshot->head); in xe_lrc_snapshot_print()
1728 snapshot->tail.internal, snapshot->tail.memory); in xe_lrc_snapshot_print()
1729 drm_printf(p, "\tRing start: (memory) 0x%08x\n", snapshot->start); in xe_lrc_snapshot_print()
1730 drm_printf(p, "\tStart seqno: (memory) %d\n", snapshot->start_seqno); in xe_lrc_snapshot_print()
1731 drm_printf(p, "\tSeqno: (memory) %d\n", snapshot->seqno); in xe_lrc_snapshot_print()
1732 drm_printf(p, "\tTimestamp: 0x%08x\n", snapshot->ctx_timestamp); in xe_lrc_snapshot_print()
1733 drm_printf(p, "\tJob Timestamp: 0x%08x\n", snapshot->ctx_job_timestamp); in xe_lrc_snapshot_print()
1735 if (!snapshot->lrc_snapshot) in xe_lrc_snapshot_print()
1741 u32 *val = snapshot->lrc_snapshot + i; in xe_lrc_snapshot_print()
1747 drm_printf(p, "\n\t[HWCTX].length: 0x%lx\n", snapshot->lrc_size - LRC_PPHWSP_SIZE); in xe_lrc_snapshot_print()
1749 for (; i < snapshot->lrc_size; i += sizeof(u32)) { in xe_lrc_snapshot_print()
1750 u32 *val = snapshot->lrc_snapshot + i; in xe_lrc_snapshot_print()
1763 kvfree(snapshot->lrc_snapshot); in xe_lrc_snapshot_free()
1764 if (snapshot->lrc_bo) { in xe_lrc_snapshot_free()
1765 struct xe_vm *vm; in xe_lrc_snapshot_free() local
1767 vm = snapshot->lrc_bo->vm; in xe_lrc_snapshot_free()
1768 xe_bo_put(snapshot->lrc_bo); in xe_lrc_snapshot_free()
1769 if (vm) in xe_lrc_snapshot_free()
1770 xe_vm_put(vm); in xe_lrc_snapshot_free()
1776 * xe_lrc_update_timestamp() - Update ctx timestamp
1787 *old_ts = lrc->ctx_timestamp; in xe_lrc_update_timestamp()
1789 lrc->ctx_timestamp = xe_lrc_ctx_timestamp(lrc); in xe_lrc_update_timestamp()
1793 return lrc->ctx_timestamp; in xe_lrc_update_timestamp()
1797 * xe_lrc_ring_is_idle() - LRC is idle