Lines Matching +full:1 +full:q
31 XE_EXEC_QUEUE_TIMESLICE = 1,
36 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q,
39 static void __xe_exec_queue_free(struct xe_exec_queue *q) in __xe_exec_queue_free() argument
41 if (q->vm) in __xe_exec_queue_free()
42 xe_vm_put(q->vm); in __xe_exec_queue_free()
44 if (q->xef) in __xe_exec_queue_free()
45 xe_file_put(q->xef); in __xe_exec_queue_free()
47 kfree(q); in __xe_exec_queue_free()
56 struct xe_exec_queue *q; in __xe_exec_queue_alloc() local
63 q = kzalloc(struct_size(q, lrc, width), GFP_KERNEL); in __xe_exec_queue_alloc()
64 if (!q) in __xe_exec_queue_alloc()
67 kref_init(&q->refcount); in __xe_exec_queue_alloc()
68 q->flags = flags; in __xe_exec_queue_alloc()
69 q->hwe = hwe; in __xe_exec_queue_alloc()
70 q->gt = gt; in __xe_exec_queue_alloc()
71 q->class = hwe->class; in __xe_exec_queue_alloc()
72 q->width = width; in __xe_exec_queue_alloc()
73 q->msix_vec = XE_IRQ_DEFAULT_MSIX; in __xe_exec_queue_alloc()
74 q->logical_mask = logical_mask; in __xe_exec_queue_alloc()
75 q->fence_irq = >->fence_irq[hwe->class]; in __xe_exec_queue_alloc()
76 q->ring_ops = gt->ring_ops[hwe->class]; in __xe_exec_queue_alloc()
77 q->ops = gt->exec_queue_ops; in __xe_exec_queue_alloc()
78 INIT_LIST_HEAD(&q->lr.link); in __xe_exec_queue_alloc()
79 INIT_LIST_HEAD(&q->multi_gt_link); in __xe_exec_queue_alloc()
80 INIT_LIST_HEAD(&q->hw_engine_group_link); in __xe_exec_queue_alloc()
82 q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us; in __xe_exec_queue_alloc()
83 q->sched_props.preempt_timeout_us = in __xe_exec_queue_alloc()
85 q->sched_props.job_timeout_ms = in __xe_exec_queue_alloc()
87 if (q->flags & EXEC_QUEUE_FLAG_KERNEL && in __xe_exec_queue_alloc()
88 q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY) in __xe_exec_queue_alloc()
89 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL; in __xe_exec_queue_alloc()
91 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL; in __xe_exec_queue_alloc()
94 q->vm = xe_vm_get(vm); in __xe_exec_queue_alloc()
98 * may set q->usm, must come before xe_lrc_create(), in __xe_exec_queue_alloc()
99 * may overwrite q->sched_props, must come before q->ops->init() in __xe_exec_queue_alloc()
101 err = exec_queue_user_extensions(xe, q, extensions, 0); in __xe_exec_queue_alloc()
103 __xe_exec_queue_free(q); in __xe_exec_queue_alloc()
108 return q; in __xe_exec_queue_alloc()
111 static int __xe_exec_queue_init(struct xe_exec_queue *q) in __xe_exec_queue_init() argument
113 struct xe_vm *vm = q->vm; in __xe_exec_queue_init()
122 for (i = 0; i < q->width; ++i) { in __xe_exec_queue_init()
123 q->lrc[i] = xe_lrc_create(q->hwe, q->vm, SZ_16K, q->msix_vec); in __xe_exec_queue_init()
124 if (IS_ERR(q->lrc[i])) { in __xe_exec_queue_init()
125 err = PTR_ERR(q->lrc[i]); in __xe_exec_queue_init()
133 err = q->ops->init(q); in __xe_exec_queue_init()
143 for (i = i - 1; i >= 0; --i) in __xe_exec_queue_init()
144 xe_lrc_put(q->lrc[i]); in __xe_exec_queue_init()
153 struct xe_exec_queue *q; in xe_exec_queue_create() local
156 q = __xe_exec_queue_alloc(xe, vm, logical_mask, width, hwe, flags, in xe_exec_queue_create()
158 if (IS_ERR(q)) in xe_exec_queue_create()
159 return q; in xe_exec_queue_create()
161 err = __xe_exec_queue_init(q); in xe_exec_queue_create()
165 return q; in xe_exec_queue_create()
168 __xe_exec_queue_free(q); in xe_exec_queue_create()
195 return xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, flags, extensions); in xe_exec_queue_create_class()
219 struct xe_exec_queue *q; in xe_exec_queue_create_bind() local
234 q = xe_exec_queue_create(xe, migrate_vm, in xe_exec_queue_create_bind()
235 BIT(hwe->logical_instance), 1, hwe, in xe_exec_queue_create_bind()
238 q = xe_exec_queue_create_class(xe, gt, migrate_vm, in xe_exec_queue_create_bind()
244 return q; in xe_exec_queue_create_bind()
250 struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount); in xe_exec_queue_destroy() local
253 xe_exec_queue_last_fence_put_unlocked(q); in xe_exec_queue_destroy()
254 if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) { in xe_exec_queue_destroy()
255 list_for_each_entry_safe(eq, next, &q->multi_gt_list, in xe_exec_queue_destroy()
260 q->ops->fini(q); in xe_exec_queue_destroy()
263 void xe_exec_queue_fini(struct xe_exec_queue *q) in xe_exec_queue_fini() argument
271 xe_exec_queue_update_run_ticks(q); in xe_exec_queue_fini()
272 if (q->xef && atomic_dec_and_test(&q->xef->exec_queue.pending_removal)) in xe_exec_queue_fini()
273 wake_up_var(&q->xef->exec_queue.pending_removal); in xe_exec_queue_fini()
275 for (i = 0; i < q->width; ++i) in xe_exec_queue_fini()
276 xe_lrc_put(q->lrc[i]); in xe_exec_queue_fini()
278 __xe_exec_queue_free(q); in xe_exec_queue_fini()
281 void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance) in xe_exec_queue_assign_name() argument
283 switch (q->class) { in xe_exec_queue_assign_name()
285 snprintf(q->name, sizeof(q->name), "rcs%d", instance); in xe_exec_queue_assign_name()
288 snprintf(q->name, sizeof(q->name), "vcs%d", instance); in xe_exec_queue_assign_name()
291 snprintf(q->name, sizeof(q->name), "vecs%d", instance); in xe_exec_queue_assign_name()
294 snprintf(q->name, sizeof(q->name), "bcs%d", instance); in xe_exec_queue_assign_name()
297 snprintf(q->name, sizeof(q->name), "ccs%d", instance); in xe_exec_queue_assign_name()
300 snprintf(q->name, sizeof(q->name), "gsccs%d", instance); in xe_exec_queue_assign_name()
303 XE_WARN_ON(q->class); in xe_exec_queue_assign_name()
309 struct xe_exec_queue *q; in xe_exec_queue_lookup() local
312 q = xa_load(&xef->exec_queue.xa, id); in xe_exec_queue_lookup()
313 if (q) in xe_exec_queue_lookup()
314 xe_exec_queue_get(q); in xe_exec_queue_lookup()
317 return q; in xe_exec_queue_lookup()
327 static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q, in exec_queue_set_priority() argument
336 q->sched_props.priority = value; in exec_queue_set_priority()
392 static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q, in exec_queue_set_timeslice() argument
397 xe_exec_queue_get_prop_minmax(q->hwe->eclass, in exec_queue_set_timeslice()
404 q->sched_props.timeslice_us = value; in exec_queue_set_timeslice()
409 struct xe_exec_queue *q,
418 struct xe_exec_queue *q, in exec_queue_user_ext_set_property() argument
441 return exec_queue_set_property_funcs[idx](xe, q, ext.value); in exec_queue_user_ext_set_property()
445 struct xe_exec_queue *q,
453 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, in exec_queue_user_extensions() argument
475 err = exec_queue_user_extension_funcs[idx](xe, q, extensions); in exec_queue_user_extensions()
480 return exec_queue_user_extensions(xe, q, ext.next_extension, in exec_queue_user_extensions()
497 len > 1)) in calc_validate_logical_mask()
522 if (width == 1 || !i) in calc_validate_logical_mask()
528 if (i && XE_IOCTL_DBG(xe, current_mask != prev_mask << 1)) in calc_validate_logical_mask()
550 struct xe_exec_queue *q = NULL; in xe_exec_queue_create_ioctl() local
557 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) in xe_exec_queue_create_ioctl()
574 if (XE_IOCTL_DBG(xe, args->width != 1) || in xe_exec_queue_create_ioctl()
575 XE_IOCTL_DBG(xe, args->num_placements != 1) || in xe_exec_queue_create_ioctl()
590 if (q) in xe_exec_queue_create_ioctl()
595 q = new; in xe_exec_queue_create_ioctl()
598 &q->multi_gt_link); in xe_exec_queue_create_ioctl()
628 q = xe_exec_queue_create(xe, vm, logical_mask, in xe_exec_queue_create_ioctl()
633 if (IS_ERR(q)) in xe_exec_queue_create_ioctl()
634 return PTR_ERR(q); in xe_exec_queue_create_ioctl()
637 q->lr.context = dma_fence_context_alloc(1); in xe_exec_queue_create_ioctl()
639 err = xe_vm_add_compute_exec_queue(vm, q); in xe_exec_queue_create_ioctl()
644 if (q->vm && q->hwe->hw_engine_group) { in xe_exec_queue_create_ioctl()
645 err = xe_hw_engine_group_add_exec_queue(q->hwe->hw_engine_group, q); in xe_exec_queue_create_ioctl()
651 q->xef = xe_file_get(xef); in xe_exec_queue_create_ioctl()
654 err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL); in xe_exec_queue_create_ioctl()
663 xe_exec_queue_kill(q); in xe_exec_queue_create_ioctl()
665 xe_exec_queue_put(q); in xe_exec_queue_create_ioctl()
675 struct xe_exec_queue *q; in xe_exec_queue_get_property_ioctl() local
678 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) in xe_exec_queue_get_property_ioctl()
681 q = xe_exec_queue_lookup(xef, args->exec_queue_id); in xe_exec_queue_get_property_ioctl()
682 if (XE_IOCTL_DBG(xe, !q)) in xe_exec_queue_get_property_ioctl()
687 args->value = q->ops->reset_status(q); in xe_exec_queue_get_property_ioctl()
694 xe_exec_queue_put(q); in xe_exec_queue_get_property_ioctl()
701 * @q: The exec_queue
705 bool xe_exec_queue_is_lr(struct xe_exec_queue *q) in xe_exec_queue_is_lr() argument
707 return q->vm && xe_vm_in_lr_mode(q->vm) && in xe_exec_queue_is_lr()
708 !(q->flags & EXEC_QUEUE_FLAG_VM); in xe_exec_queue_is_lr()
711 static s32 xe_exec_queue_num_job_inflight(struct xe_exec_queue *q) in xe_exec_queue_num_job_inflight() argument
713 return q->lrc[0]->fence_ctx.next_seqno - xe_lrc_seqno(q->lrc[0]) - 1; in xe_exec_queue_num_job_inflight()
718 * @q: The exec_queue
722 bool xe_exec_queue_ring_full(struct xe_exec_queue *q) in xe_exec_queue_ring_full() argument
724 struct xe_lrc *lrc = q->lrc[0]; in xe_exec_queue_ring_full()
727 return xe_exec_queue_num_job_inflight(q) >= max_job; in xe_exec_queue_ring_full()
732 * @q: The exec_queue
744 bool xe_exec_queue_is_idle(struct xe_exec_queue *q) in xe_exec_queue_is_idle() argument
746 if (xe_exec_queue_is_parallel(q)) { in xe_exec_queue_is_idle()
749 for (i = 0; i < q->width; ++i) { in xe_exec_queue_is_idle()
750 if (xe_lrc_seqno(q->lrc[i]) != in xe_exec_queue_is_idle()
751 q->lrc[i]->fence_ctx.next_seqno - 1) in xe_exec_queue_is_idle()
758 return xe_lrc_seqno(q->lrc[0]) == in xe_exec_queue_is_idle()
759 q->lrc[0]->fence_ctx.next_seqno - 1; in xe_exec_queue_is_idle()
765 * @q: The exec queue
770 void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q) in xe_exec_queue_update_run_ticks() argument
772 struct xe_device *xe = gt_to_xe(q->gt); in xe_exec_queue_update_run_ticks()
781 if (!q->xef) in xe_exec_queue_update_run_ticks()
795 lrc = q->lrc[0]; in xe_exec_queue_update_run_ticks()
797 q->xef->run_ticks[q->class] += (new_ts - old_ts) * q->width; in xe_exec_queue_update_run_ticks()
804 * @q: The exec queue
811 void xe_exec_queue_kill(struct xe_exec_queue *q) in xe_exec_queue_kill() argument
813 struct xe_exec_queue *eq = q, *next; in xe_exec_queue_kill()
817 q->ops->kill(eq); in xe_exec_queue_kill()
818 xe_vm_remove_compute_exec_queue(q->vm, eq); in xe_exec_queue_kill()
821 q->ops->kill(q); in xe_exec_queue_kill()
822 xe_vm_remove_compute_exec_queue(q->vm, q); in xe_exec_queue_kill()
831 struct xe_exec_queue *q; in xe_exec_queue_destroy_ioctl() local
834 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) in xe_exec_queue_destroy_ioctl()
838 q = xa_erase(&xef->exec_queue.xa, args->exec_queue_id); in xe_exec_queue_destroy_ioctl()
839 if (q) in xe_exec_queue_destroy_ioctl()
843 if (XE_IOCTL_DBG(xe, !q)) in xe_exec_queue_destroy_ioctl()
846 if (q->vm && q->hwe->hw_engine_group) in xe_exec_queue_destroy_ioctl()
847 xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q); in xe_exec_queue_destroy_ioctl()
849 xe_exec_queue_kill(q); in xe_exec_queue_destroy_ioctl()
851 trace_xe_exec_queue_close(q); in xe_exec_queue_destroy_ioctl()
852 xe_exec_queue_put(q); in xe_exec_queue_destroy_ioctl()
857 static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q, in xe_exec_queue_last_fence_lockdep_assert() argument
860 if (q->flags & EXEC_QUEUE_FLAG_VM) { in xe_exec_queue_last_fence_lockdep_assert()
864 lockdep_assert_held(&q->hwe->hw_engine_group->mode_sem); in xe_exec_queue_last_fence_lockdep_assert()
870 * @q: The exec queue
873 void xe_exec_queue_last_fence_put(struct xe_exec_queue *q, struct xe_vm *vm) in xe_exec_queue_last_fence_put() argument
875 xe_exec_queue_last_fence_lockdep_assert(q, vm); in xe_exec_queue_last_fence_put()
877 xe_exec_queue_last_fence_put_unlocked(q); in xe_exec_queue_last_fence_put()
882 * @q: The exec queue
886 void xe_exec_queue_last_fence_put_unlocked(struct xe_exec_queue *q) in xe_exec_queue_last_fence_put_unlocked() argument
888 if (q->last_fence) { in xe_exec_queue_last_fence_put_unlocked()
889 dma_fence_put(q->last_fence); in xe_exec_queue_last_fence_put_unlocked()
890 q->last_fence = NULL; in xe_exec_queue_last_fence_put_unlocked()
896 * @q: The exec queue
903 struct dma_fence *xe_exec_queue_last_fence_get(struct xe_exec_queue *q, in xe_exec_queue_last_fence_get() argument
908 xe_exec_queue_last_fence_lockdep_assert(q, vm); in xe_exec_queue_last_fence_get()
910 if (q->last_fence && in xe_exec_queue_last_fence_get()
911 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) in xe_exec_queue_last_fence_get()
912 xe_exec_queue_last_fence_put(q, vm); in xe_exec_queue_last_fence_get()
914 fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); in xe_exec_queue_last_fence_get()
921 * @q: The exec queue
930 struct dma_fence *xe_exec_queue_last_fence_get_for_resume(struct xe_exec_queue *q, in xe_exec_queue_last_fence_get_for_resume() argument
935 lockdep_assert_held_write(&q->hwe->hw_engine_group->mode_sem); in xe_exec_queue_last_fence_get_for_resume()
937 if (q->last_fence && in xe_exec_queue_last_fence_get_for_resume()
938 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) in xe_exec_queue_last_fence_get_for_resume()
939 xe_exec_queue_last_fence_put_unlocked(q); in xe_exec_queue_last_fence_get_for_resume()
941 fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); in xe_exec_queue_last_fence_get_for_resume()
948 * @q: The exec queue
955 void xe_exec_queue_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm, in xe_exec_queue_last_fence_set() argument
958 xe_exec_queue_last_fence_lockdep_assert(q, vm); in xe_exec_queue_last_fence_set()
960 xe_exec_queue_last_fence_put(q, vm); in xe_exec_queue_last_fence_set()
961 q->last_fence = dma_fence_get(fence); in xe_exec_queue_last_fence_set()
966 * @q: The exec queue
972 int xe_exec_queue_last_fence_test_dep(struct xe_exec_queue *q, struct xe_vm *vm) in xe_exec_queue_last_fence_test_dep() argument
977 fence = xe_exec_queue_last_fence_get(q, vm); in xe_exec_queue_last_fence_test_dep()