Lines Matching +full:irq +full:- +full:signals
1 // SPDX-License-Identifier: MIT
3 * Copyright © 2015-2021 Intel Corporation
21 return intel_engine_irq_enable(b->irq_engine); in irq_enable()
26 intel_engine_irq_disable(b->irq_engine); in irq_disable()
37 wakeref = intel_gt_pm_get_if_awake(b->irq_engine->gt); in __intel_breadcrumbs_arm_irq()
42 * The breadcrumb irq will be disarmed on the interrupt after the in __intel_breadcrumbs_arm_irq()
44 * which we can add a new waiter and avoid the cost of re-enabling in __intel_breadcrumbs_arm_irq()
45 * the irq. in __intel_breadcrumbs_arm_irq()
47 WRITE_ONCE(b->irq_armed, wakeref); in __intel_breadcrumbs_arm_irq()
50 if (!b->irq_enabled++ && b->irq_enable(b)) in __intel_breadcrumbs_arm_irq()
51 irq_work_queue(&b->irq_work); in __intel_breadcrumbs_arm_irq()
56 if (!b->irq_engine) in intel_breadcrumbs_arm_irq()
59 spin_lock(&b->irq_lock); in intel_breadcrumbs_arm_irq()
60 if (!b->irq_armed) in intel_breadcrumbs_arm_irq()
62 spin_unlock(&b->irq_lock); in intel_breadcrumbs_arm_irq()
67 intel_wakeref_t wakeref = b->irq_armed; in __intel_breadcrumbs_disarm_irq()
69 GEM_BUG_ON(!b->irq_enabled); in __intel_breadcrumbs_disarm_irq()
70 if (!--b->irq_enabled) in __intel_breadcrumbs_disarm_irq()
71 b->irq_disable(b); in __intel_breadcrumbs_disarm_irq()
73 WRITE_ONCE(b->irq_armed, NULL); in __intel_breadcrumbs_disarm_irq()
74 intel_gt_pm_put_async(b->irq_engine->gt, wakeref); in __intel_breadcrumbs_disarm_irq()
79 spin_lock(&b->irq_lock); in intel_breadcrumbs_disarm_irq()
80 if (b->irq_armed) in intel_breadcrumbs_disarm_irq()
82 spin_unlock(&b->irq_lock); in intel_breadcrumbs_disarm_irq()
88 lockdep_assert_held(&ce->signal_lock); in add_signaling_context()
90 spin_lock(&b->signalers_lock); in add_signaling_context()
91 list_add_rcu(&ce->signal_link, &b->signalers); in add_signaling_context()
92 spin_unlock(&b->signalers_lock); in add_signaling_context()
98 lockdep_assert_held(&ce->signal_lock); in remove_signaling_context()
100 if (!list_empty(&ce->signals)) in remove_signaling_context()
103 spin_lock(&b->signalers_lock); in remove_signaling_context()
104 list_del_rcu(&ce->signal_link); in remove_signaling_context()
105 spin_unlock(&b->signalers_lock); in remove_signaling_context()
113 if (rq->context != ce) in check_signal_order()
116 if (!list_is_last(&rq->signal_link, &ce->signals) && in check_signal_order()
117 i915_seqno_passed(rq->fence.seqno, in check_signal_order()
118 list_next_entry(rq, signal_link)->fence.seqno)) in check_signal_order()
121 if (!list_is_first(&rq->signal_link, &ce->signals) && in check_signal_order()
122 i915_seqno_passed(list_prev_entry(rq, signal_link)->fence.seqno, in check_signal_order()
123 rq->fence.seqno)) in check_signal_order()
132 return !test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags); in __dma_fence_signal()
138 fence->timestamp = timestamp; in __dma_fence_signal__timestamp()
139 set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags); in __dma_fence_signal__timestamp()
149 lockdep_assert_held(fence->lock); in __dma_fence_signal__notify()
152 INIT_LIST_HEAD(&cur->node); in __dma_fence_signal__notify()
153 cur->func(fence, cur); in __dma_fence_signal__notify()
159 if (b->irq_engine) in add_retire()
160 intel_engine_add_retire(b->irq_engine, tl); in add_retire()
166 node->next = head; in slist_add()
178 if (unlikely(!llist_empty(&b->signaled_requests))) in signal_irq_work()
179 signal = llist_del_all(&b->signaled_requests); in signal_irq_work()
182 * Keep the irq armed until the interrupt after all listeners are gone. in signal_irq_work()
196 * dysfunctional with per-engine interrupt masking. in signal_irq_work()
202 * Fewer interrupts should conserve power -- at the very least, fewer in signal_irq_work()
206 if (!signal && READ_ONCE(b->irq_armed) && list_empty(&b->signalers)) in signal_irq_work()
210 atomic_inc(&b->signaler_active); in signal_irq_work()
211 list_for_each_entry_rcu(ce, &b->signalers, signal_link) { in signal_irq_work()
214 list_for_each_entry_rcu(rq, &ce->signals, signal_link) { in signal_irq_work()
221 &rq->fence.flags)) in signal_irq_work()
229 spin_lock(&ce->signal_lock); in signal_irq_work()
230 list_del_rcu(&rq->signal_link); in signal_irq_work()
232 spin_unlock(&ce->signal_lock); in signal_irq_work()
234 if (intel_timeline_is_last(ce->timeline, rq)) in signal_irq_work()
235 add_retire(b, ce->timeline); in signal_irq_work()
239 if (__dma_fence_signal(&rq->fence)) in signal_irq_work()
241 signal = slist_add(&rq->signal_node, signal); in signal_irq_work()
246 atomic_dec(&b->signaler_active); in signal_irq_work()
254 if (rq->engine->sched_engine->retire_inflight_request_prio) in signal_irq_work()
255 rq->engine->sched_engine->retire_inflight_request_prio(rq); in signal_irq_work()
257 spin_lock(&rq->lock); in signal_irq_work()
258 list_replace(&rq->fence.cb_list, &cb_list); in signal_irq_work()
259 __dma_fence_signal__timestamp(&rq->fence, timestamp); in signal_irq_work()
260 __dma_fence_signal__notify(&rq->fence, &cb_list); in signal_irq_work()
261 spin_unlock(&rq->lock); in signal_irq_work()
266 /* Lazy irq enabling after HW submission */ in signal_irq_work()
267 if (!READ_ONCE(b->irq_armed) && !list_empty(&b->signalers)) in signal_irq_work()
271 if (READ_ONCE(b->irq_armed) && !atomic_read(&b->active)) in signal_irq_work()
284 kref_init(&b->ref); in intel_breadcrumbs_create()
286 spin_lock_init(&b->signalers_lock); in intel_breadcrumbs_create()
287 INIT_LIST_HEAD(&b->signalers); in intel_breadcrumbs_create()
288 init_llist_head(&b->signaled_requests); in intel_breadcrumbs_create()
290 spin_lock_init(&b->irq_lock); in intel_breadcrumbs_create()
291 init_irq_work(&b->irq_work, signal_irq_work); in intel_breadcrumbs_create()
293 b->irq_engine = irq_engine; in intel_breadcrumbs_create()
294 b->irq_enable = irq_enable; in intel_breadcrumbs_create()
295 b->irq_disable = irq_disable; in intel_breadcrumbs_create()
304 if (!b->irq_engine) in intel_breadcrumbs_reset()
307 spin_lock_irqsave(&b->irq_lock, flags); in intel_breadcrumbs_reset()
309 if (b->irq_enabled) in intel_breadcrumbs_reset()
310 b->irq_enable(b); in intel_breadcrumbs_reset()
312 b->irq_disable(b); in intel_breadcrumbs_reset()
314 spin_unlock_irqrestore(&b->irq_lock, flags); in intel_breadcrumbs_reset()
319 if (!READ_ONCE(b->irq_armed)) in __intel_breadcrumbs_park()
322 /* Kick the work once more to drain the signalers, and disarm the irq */ in __intel_breadcrumbs_park()
323 irq_work_queue(&b->irq_work); in __intel_breadcrumbs_park()
330 irq_work_sync(&b->irq_work); in intel_breadcrumbs_free()
331 GEM_BUG_ON(!list_empty(&b->signalers)); in intel_breadcrumbs_free()
332 GEM_BUG_ON(b->irq_armed); in intel_breadcrumbs_free()
340 if (!__dma_fence_signal(&rq->fence)) in irq_signal_request()
344 if (llist_add(&rq->signal_node, &b->signaled_requests)) in irq_signal_request()
345 irq_work_queue(&b->irq_work); in irq_signal_request()
350 struct intel_breadcrumbs *b = READ_ONCE(rq->engine)->breadcrumbs; in insert_breadcrumb()
351 struct intel_context *ce = rq->context; in insert_breadcrumb()
354 if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) in insert_breadcrumb()
359 * straight onto a signaled list, and queue the irq worker for in insert_breadcrumb()
367 if (list_empty(&ce->signals)) { in insert_breadcrumb()
370 pos = &ce->signals; in insert_breadcrumb()
376 * hasn't event started). We could walk the timeline->requests, in insert_breadcrumb()
386 list_for_each_prev(pos, &ce->signals) { in insert_breadcrumb()
390 if (i915_seqno_passed(rq->fence.seqno, it->fence.seqno)) in insert_breadcrumb()
396 list_add_rcu(&rq->signal_link, pos); in insert_breadcrumb()
398 GEM_BUG_ON(test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags)); in insert_breadcrumb()
399 set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); in insert_breadcrumb()
406 if (!READ_ONCE(b->irq_armed) || __i915_request_is_complete(rq)) in insert_breadcrumb()
407 irq_work_queue(&b->irq_work); in insert_breadcrumb()
412 struct intel_context *ce = rq->context; in i915_request_enable_breadcrumb()
414 /* Serialises with i915_request_retire() using rq->lock */ in i915_request_enable_breadcrumb()
415 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags)) in i915_request_enable_breadcrumb()
424 if (!test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) in i915_request_enable_breadcrumb()
427 spin_lock(&ce->signal_lock); in i915_request_enable_breadcrumb()
428 if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) in i915_request_enable_breadcrumb()
430 spin_unlock(&ce->signal_lock); in i915_request_enable_breadcrumb()
437 struct intel_breadcrumbs *b = READ_ONCE(rq->engine)->breadcrumbs; in i915_request_cancel_breadcrumb()
438 struct intel_context *ce = rq->context; in i915_request_cancel_breadcrumb()
441 spin_lock(&ce->signal_lock); in i915_request_cancel_breadcrumb()
442 if (!test_and_clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) { in i915_request_cancel_breadcrumb()
443 spin_unlock(&ce->signal_lock); in i915_request_cancel_breadcrumb()
447 list_del_rcu(&rq->signal_link); in i915_request_cancel_breadcrumb()
449 spin_unlock(&ce->signal_lock); in i915_request_cancel_breadcrumb()
466 spin_lock_irqsave(&ce->signal_lock, flags); in intel_context_remove_breadcrumbs()
468 if (list_empty(&ce->signals)) in intel_context_remove_breadcrumbs()
471 list_for_each_entry_safe(rq, rn, &ce->signals, signal_link) { in intel_context_remove_breadcrumbs()
474 &rq->fence.flags)) in intel_context_remove_breadcrumbs()
477 list_del_rcu(&rq->signal_link); in intel_context_remove_breadcrumbs()
484 spin_unlock_irqrestore(&ce->signal_lock, flags); in intel_context_remove_breadcrumbs()
488 while (atomic_read(&b->signaler_active)) in intel_context_remove_breadcrumbs()
497 drm_printf(p, "Signals:\n"); in print_signals()
500 list_for_each_entry_rcu(ce, &b->signalers, signal_link) { in print_signals()
501 list_for_each_entry_rcu(rq, &ce->signals, signal_link) in print_signals()
503 rq->fence.context, rq->fence.seqno, in print_signals()
507 jiffies_to_msecs(jiffies - rq->emitted_jiffies)); in print_signals()
517 b = engine->breadcrumbs; in intel_engine_print_breadcrumbs()
521 drm_printf(p, "IRQ: %s\n", str_enabled_disabled(b->irq_armed)); in intel_engine_print_breadcrumbs()
522 if (!list_empty(&b->signalers)) in intel_engine_print_breadcrumbs()