Lines Matching +full:panfrost +full:- +full:gpu
1 // SPDX-License-Identifier: GPL-2.0
10 #include <linux/dma-resv.h>
27 #define job_write(dev, reg, data) writel(data, dev->iomem + (reg))
28 #define job_read(dev, reg) readl(dev->iomem + (reg))
51 /* panfrost seqno for signaled() test */
64 return "panfrost"; in panfrost_fence_get_driver_name()
71 switch (f->queue) { in panfrost_fence_get_timeline_name()
73 return "panfrost-js-0"; in panfrost_fence_get_timeline_name()
75 return "panfrost-js-1"; in panfrost_fence_get_timeline_name()
77 return "panfrost-js-2"; in panfrost_fence_get_timeline_name()
91 struct panfrost_job_slot *js = pfdev->js; in panfrost_fence_create()
95 return ERR_PTR(-ENOMEM); in panfrost_fence_create()
97 fence->dev = pfdev->ddev; in panfrost_fence_create()
98 fence->queue = js_num; in panfrost_fence_create()
99 fence->seqno = ++js->queue[js_num].emit_seqno; in panfrost_fence_create()
100 dma_fence_init(&fence->base, &panfrost_fence_ops, &js->job_lock, in panfrost_fence_create()
101 js->queue[js_num].fence_context, fence->seqno); in panfrost_fence_create()
103 return &fence->base; in panfrost_fence_create()
112 if (job->requirements & PANFROST_JD_REQ_FS) in panfrost_job_get_slot()
117 if (job->requirements & PANFROST_JD_REQ_ONLY_COMPUTE) { in panfrost_job_get_slot()
118 if ((job->requirements & PANFROST_JD_REQ_CORE_GRP_MASK) && in panfrost_job_get_slot()
119 (job->pfdev->features.nr_core_groups == 2)) in panfrost_job_get_slot()
121 if (panfrost_has_hw_issue(job->pfdev, HW_ISSUE_8987)) in panfrost_job_get_slot()
139 affinity = pfdev->features.shader_present; in panfrost_job_write_affinity()
148 struct panfrost_fence *f = to_panfrost_fence(job->done_fence); in panfrost_get_job_chain_flag()
150 if (!panfrost_has_hw_feature(job->pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) in panfrost_get_job_chain_flag()
153 return (f->seqno & 1) ? JS_CONFIG_JOB_CHAIN_FLAG : 0; in panfrost_get_job_chain_flag()
159 struct panfrost_job *job = pfdev->jobs[slot][0]; in panfrost_dequeue_job()
163 if (job->is_profiled && job->engine_usage) { in panfrost_dequeue_job()
164 job->engine_usage->elapsed_ns[slot] += in panfrost_dequeue_job()
165 ktime_to_ns(ktime_sub(ktime_get(), job->start_time)); in panfrost_dequeue_job()
166 job->engine_usage->cycles[slot] += in panfrost_dequeue_job()
167 panfrost_cycle_counter_read(pfdev) - job->start_cycles; in panfrost_dequeue_job()
170 if (job->requirements & PANFROST_JD_REQ_CYCLE_COUNT || job->is_profiled) in panfrost_dequeue_job()
173 pfdev->jobs[slot][0] = pfdev->jobs[slot][1]; in panfrost_dequeue_job()
174 pfdev->jobs[slot][1] = NULL; in panfrost_dequeue_job()
186 if (!pfdev->jobs[slot][0]) { in panfrost_enqueue_job()
187 pfdev->jobs[slot][0] = job; in panfrost_enqueue_job()
191 WARN_ON(pfdev->jobs[slot][1]); in panfrost_enqueue_job()
192 pfdev->jobs[slot][1] = job; in panfrost_enqueue_job()
194 panfrost_get_job_chain_flag(pfdev->jobs[slot][0])); in panfrost_enqueue_job()
200 struct panfrost_device *pfdev = job->pfdev; in panfrost_job_hw_submit()
203 u64 jc_head = job->jc; in panfrost_job_hw_submit()
206 panfrost_devfreq_record_busy(&pfdev->pfdevfreq); in panfrost_job_hw_submit()
208 ret = pm_runtime_get_sync(pfdev->dev); in panfrost_job_hw_submit()
216 cfg = panfrost_mmu_as_get(pfdev, job->mmu); in panfrost_job_hw_submit()
221 panfrost_job_write_affinity(pfdev, job->requirements, js); in panfrost_job_hw_submit()
239 job_write(pfdev, JS_FLUSH_ID_NEXT(js), job->flush_id); in panfrost_job_hw_submit()
243 spin_lock(&pfdev->js->job_lock); in panfrost_job_hw_submit()
246 if (!atomic_read(&pfdev->reset.pending)) { in panfrost_job_hw_submit()
247 job->is_profiled = pfdev->profile_mode; in panfrost_job_hw_submit()
249 if (job->requirements & PANFROST_JD_REQ_CYCLE_COUNT || in panfrost_job_hw_submit()
250 job->is_profiled) in panfrost_job_hw_submit()
253 if (job->is_profiled) { in panfrost_job_hw_submit()
254 job->start_time = ktime_get(); in panfrost_job_hw_submit()
255 job->start_cycles = panfrost_cycle_counter_read(pfdev); in panfrost_job_hw_submit()
259 dev_dbg(pfdev->dev, in panfrost_job_hw_submit()
263 spin_unlock(&pfdev->js->job_lock); in panfrost_job_hw_submit()
273 ret = dma_resv_reserve_fences(bos[i]->resv, 1); in panfrost_acquire_object_fences()
277 /* panfrost always uses write mode in its current uapi */ in panfrost_acquire_object_fences()
294 dma_resv_add_fence(bos[i]->resv, fence, DMA_RESV_USAGE_WRITE); in panfrost_attach_object_fences()
299 struct panfrost_device *pfdev = job->pfdev; in panfrost_job_push()
303 ret = drm_gem_lock_reservations(job->bos, job->bo_count, in panfrost_job_push()
308 mutex_lock(&pfdev->sched_lock); in panfrost_job_push()
309 drm_sched_job_arm(&job->base); in panfrost_job_push()
311 job->render_done_fence = dma_fence_get(&job->base.s_fence->finished); in panfrost_job_push()
313 ret = panfrost_acquire_object_fences(job->bos, job->bo_count, in panfrost_job_push()
314 &job->base); in panfrost_job_push()
316 mutex_unlock(&pfdev->sched_lock); in panfrost_job_push()
320 kref_get(&job->refcount); /* put by scheduler job completion */ in panfrost_job_push()
322 drm_sched_entity_push_job(&job->base); in panfrost_job_push()
324 mutex_unlock(&pfdev->sched_lock); in panfrost_job_push()
326 panfrost_attach_object_fences(job->bos, job->bo_count, in panfrost_job_push()
327 job->render_done_fence); in panfrost_job_push()
330 drm_gem_unlock_reservations(job->bos, job->bo_count, &acquire_ctx); in panfrost_job_push()
341 dma_fence_put(job->done_fence); in panfrost_job_cleanup()
342 dma_fence_put(job->render_done_fence); in panfrost_job_cleanup()
344 if (job->mappings) { in panfrost_job_cleanup()
345 for (i = 0; i < job->bo_count; i++) { in panfrost_job_cleanup()
346 if (!job->mappings[i]) in panfrost_job_cleanup()
349 atomic_dec(&job->mappings[i]->obj->gpu_usecount); in panfrost_job_cleanup()
350 panfrost_gem_mapping_put(job->mappings[i]); in panfrost_job_cleanup()
352 kvfree(job->mappings); in panfrost_job_cleanup()
355 if (job->bos) { in panfrost_job_cleanup()
356 for (i = 0; i < job->bo_count; i++) in panfrost_job_cleanup()
357 drm_gem_object_put(job->bos[i]); in panfrost_job_cleanup()
359 kvfree(job->bos); in panfrost_job_cleanup()
367 kref_put(&job->refcount, panfrost_job_cleanup); in panfrost_job_put()
382 struct panfrost_device *pfdev = job->pfdev; in panfrost_job_run()
386 if (unlikely(job->base.s_fence->finished.error)) in panfrost_job_run()
390 * we were resetting the GPU. in panfrost_job_run()
392 if (!job->jc) in panfrost_job_run()
399 if (job->done_fence) in panfrost_job_run()
400 dma_fence_put(job->done_fence); in panfrost_job_run()
401 job->done_fence = dma_fence_get(fence); in panfrost_job_run()
413 clear_bit(PANFROST_COMP_BIT_JOB, pfdev->is_suspended); in panfrost_job_enable_interrupts()
425 set_bit(PANFROST_COMP_BIT_JOB, pfdev->is_suspended); in panfrost_job_suspend_irq()
428 synchronize_irq(pfdev->js->irq); in panfrost_job_suspend_irq()
440 dev_dbg(pfdev->dev, "js event, js=%d, status=%s, head=0x%x, tail=0x%x", in panfrost_job_handle_err()
445 dev_err(pfdev->dev, "js fault, js=%d, status=%s, head=0x%x, tail=0x%x", in panfrost_job_handle_err()
453 job->jc = job_read(pfdev, JS_TAIL_LO(js)) | in panfrost_job_handle_err()
459 /* Job has been hard-stopped, flag it as canceled */ in panfrost_job_handle_err()
460 dma_fence_set_error(job->done_fence, -ECANCELED); in panfrost_job_handle_err()
461 job->jc = 0; in panfrost_job_handle_err()
463 /* We might want to provide finer-grained error code based on in panfrost_job_handle_err()
467 dma_fence_set_error(job->done_fence, -EINVAL); in panfrost_job_handle_err()
468 job->jc = 0; in panfrost_job_handle_err()
471 panfrost_mmu_as_put(pfdev, job->mmu); in panfrost_job_handle_err()
472 panfrost_devfreq_record_idle(&pfdev->pfdevfreq); in panfrost_job_handle_err()
475 dma_fence_signal_locked(job->done_fence); in panfrost_job_handle_err()
477 pm_runtime_put_autosuspend(pfdev->dev); in panfrost_job_handle_err()
480 atomic_set(&pfdev->reset.pending, 1); in panfrost_job_handle_err()
481 drm_sched_fault(&pfdev->js->queue[js].sched); in panfrost_job_handle_err()
488 /* Set ->jc to 0 to avoid re-submitting an already finished job (can in panfrost_job_handle_done()
489 * happen when we receive the DONE interrupt while doing a GPU reset). in panfrost_job_handle_done()
491 job->jc = 0; in panfrost_job_handle_done()
492 panfrost_mmu_as_put(pfdev, job->mmu); in panfrost_job_handle_done()
493 panfrost_devfreq_record_idle(&pfdev->pfdevfreq); in panfrost_job_handle_done()
495 dma_fence_signal_locked(job->done_fence); in panfrost_job_handle_done()
496 pm_runtime_put_autosuspend(pfdev->dev); in panfrost_job_handle_done()
557 } else if (pfdev->jobs[j][0] && !(js_state & MK_JS_MASK(j))) { in panfrost_job_handle_irq()
562 * are inactive, but one job remains in pfdev->jobs[j], in panfrost_job_handle_irq()
584 if (!failed[j] || !pfdev->jobs[j][0]) in panfrost_job_handle_irq()
587 if (pfdev->jobs[j][0]->jc == 0) { in panfrost_job_handle_irq()
591 dma_fence_set_error(canceled->done_fence, -ECANCELED); in panfrost_job_handle_irq()
593 } else if (!atomic_read(&pfdev->reset.pending)) { in panfrost_job_handle_irq()
605 pm_runtime_mark_last_busy(pfdev->dev); in panfrost_job_handle_irqs()
607 spin_lock(&pfdev->js->job_lock); in panfrost_job_handle_irqs()
609 spin_unlock(&pfdev->js->job_lock); in panfrost_job_handle_irqs()
644 if (!atomic_read(&pfdev->reset.pending)) in panfrost_reset()
661 drm_sched_stop(&pfdev->js->queue[i].sched, bad); in panfrost_reset()
672 synchronize_irq(pfdev->js->irq); in panfrost_reset()
675 /* Cancel the next job and soft-stop the running job. */ in panfrost_reset()
680 /* Wait at most 10ms for soft-stops to complete */ in panfrost_reset()
681 ret = readl_poll_timeout(pfdev->iomem + JOB_INT_JS_STATE, js_state, in panfrost_reset()
686 dev_err(pfdev->dev, "Soft-stop failed\n"); in panfrost_reset()
698 spin_lock(&pfdev->js->job_lock); in panfrost_reset()
700 for (j = 0; j < ARRAY_SIZE(pfdev->jobs[0]) && pfdev->jobs[i][j]; j++) { in panfrost_reset()
701 if (pfdev->jobs[i][j]->requirements & PANFROST_JD_REQ_CYCLE_COUNT || in panfrost_reset()
702 pfdev->jobs[i][j]->is_profiled) in panfrost_reset()
703 panfrost_cycle_counter_put(pfdev->jobs[i][j]->pfdev); in panfrost_reset()
704 pm_runtime_put_noidle(pfdev->dev); in panfrost_reset()
705 panfrost_devfreq_record_idle(&pfdev->pfdevfreq); in panfrost_reset()
708 memset(pfdev->jobs, 0, sizeof(pfdev->jobs)); in panfrost_reset()
709 spin_unlock(&pfdev->js->job_lock); in panfrost_reset()
719 /* GPU has been reset, we can clear the reset pending bit. */ in panfrost_reset()
720 atomic_set(&pfdev->reset.pending, 0); in panfrost_reset()
731 drm_sched_resubmit_jobs(&pfdev->js->queue[i].sched); in panfrost_reset()
736 drm_sched_start(&pfdev->js->queue[i].sched, 0); in panfrost_reset()
738 /* Re-enable job interrupts now that everything has been restarted. */ in panfrost_reset()
740 GENMASK(16 + NUM_JOB_SLOTS - 1, 16) | in panfrost_reset()
741 GENMASK(NUM_JOB_SLOTS - 1, 0)); in panfrost_reset()
750 struct panfrost_device *pfdev = job->pfdev; in panfrost_job_timedout()
754 * If the GPU managed to complete this jobs fence, the timeout is in panfrost_job_timedout()
757 if (dma_fence_is_signaled(job->done_fence)) in panfrost_job_timedout()
761 * Panfrost IRQ handler may take a long time to process an interrupt in panfrost_job_timedout()
765 * In order to catch such cases and not report spurious Panfrost in panfrost_job_timedout()
766 * job timeouts, synchronize the IRQ handler and re-check the fence in panfrost_job_timedout()
769 synchronize_irq(pfdev->js->irq); in panfrost_job_timedout()
771 if (dma_fence_is_signaled(job->done_fence)) { in panfrost_job_timedout()
772 dev_warn(pfdev->dev, "unexpectedly high interrupt latency\n"); in panfrost_job_timedout()
776 …dev_err(pfdev->dev, "gpu sched timeout, js=%d, config=0x%x, status=0x%x, head=0x%x, tail=0x%x, sch… in panfrost_job_timedout()
786 atomic_set(&pfdev->reset.pending, 1); in panfrost_job_timedout()
813 if (!test_bit(PANFROST_COMP_BIT_JOB, pfdev->is_suspended)) in panfrost_job_irq_handler_thread()
815 GENMASK(16 + NUM_JOB_SLOTS - 1, 16) | in panfrost_job_irq_handler_thread()
816 GENMASK(NUM_JOB_SLOTS - 1, 0)); in panfrost_job_irq_handler_thread()
826 if (test_bit(PANFROST_COMP_BIT_JOB, pfdev->is_suspended)) in panfrost_job_irq_handler()
850 pfdev->js = js = devm_kzalloc(pfdev->dev, sizeof(*js), GFP_KERNEL); in panfrost_job_init()
852 return -ENOMEM; in panfrost_job_init()
854 INIT_WORK(&pfdev->reset.work, panfrost_reset_work); in panfrost_job_init()
855 spin_lock_init(&js->job_lock); in panfrost_job_init()
857 js->irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "job"); in panfrost_job_init()
858 if (js->irq < 0) in panfrost_job_init()
859 return js->irq; in panfrost_job_init()
861 ret = devm_request_threaded_irq(pfdev->dev, js->irq, in panfrost_job_init()
864 IRQF_SHARED, KBUILD_MODNAME "-job", in panfrost_job_init()
867 dev_err(pfdev->dev, "failed to request job irq"); in panfrost_job_init()
871 pfdev->reset.wq = alloc_ordered_workqueue("panfrost-reset", 0); in panfrost_job_init()
872 if (!pfdev->reset.wq) in panfrost_job_init()
873 return -ENOMEM; in panfrost_job_init()
876 js->queue[j].fence_context = dma_fence_context_alloc(1); in panfrost_job_init()
878 ret = drm_sched_init(&js->queue[j].sched, in panfrost_job_init()
883 pfdev->reset.wq, in panfrost_job_init()
884 NULL, "pan_js", pfdev->dev); in panfrost_job_init()
886 dev_err(pfdev->dev, "Failed to create scheduler: %d.", ret); in panfrost_job_init()
896 for (j--; j >= 0; j--) in panfrost_job_init()
897 drm_sched_fini(&js->queue[j].sched); in panfrost_job_init()
899 destroy_workqueue(pfdev->reset.wq); in panfrost_job_init()
905 struct panfrost_job_slot *js = pfdev->js; in panfrost_job_fini()
911 drm_sched_fini(&js->queue[j].sched); in panfrost_job_fini()
914 cancel_work_sync(&pfdev->reset.work); in panfrost_job_fini()
915 destroy_workqueue(pfdev->reset.wq); in panfrost_job_fini()
920 struct panfrost_device *pfdev = panfrost_priv->pfdev; in panfrost_job_open()
921 struct panfrost_job_slot *js = pfdev->js; in panfrost_job_open()
926 sched = &js->queue[i].sched; in panfrost_job_open()
927 ret = drm_sched_entity_init(&panfrost_priv->sched_entity[i], in panfrost_job_open()
938 struct panfrost_device *pfdev = panfrost_priv->pfdev; in panfrost_job_close()
942 drm_sched_entity_destroy(&panfrost_priv->sched_entity[i]); in panfrost_job_close()
944 /* Kill in-flight jobs */ in panfrost_job_close()
945 spin_lock(&pfdev->js->job_lock); in panfrost_job_close()
947 struct drm_sched_entity *entity = &panfrost_priv->sched_entity[i]; in panfrost_job_close()
950 for (j = ARRAY_SIZE(pfdev->jobs[0]) - 1; j >= 0; j--) { in panfrost_job_close()
951 struct panfrost_job *job = pfdev->jobs[i][j]; in panfrost_job_close()
954 if (!job || job->base.entity != entity) in panfrost_job_close()
963 job->jc = 0; in panfrost_job_close()
977 job->engine_usage = NULL; in panfrost_job_close()
980 spin_unlock(&pfdev->js->job_lock); in panfrost_job_close()
985 struct panfrost_job_slot *js = pfdev->js; in panfrost_job_is_idle()
990 if (atomic_read(&js->queue[i].sched.credit_count)) in panfrost_job_is_idle()