106 int panfrost_job_get_slot(struct panfrost_job *job)  in panfrost_job_get_slot()  argument
112 	if (job->requirements & PANFROST_JD_REQ_FS)  in panfrost_job_get_slot()
117 	if (job->requirements & PANFROST_JD_REQ_ONLY_COMPUTE) {  in panfrost_job_get_slot()
118 		if ((job->requirements & PANFROST_JD_REQ_CORE_GRP_MASK) &&  in panfrost_job_get_slot()
119 		    (job->pfdev->features.nr_core_groups == 2))  in panfrost_job_get_slot()
121 		if (panfrost_has_hw_issue(job->pfdev, HW_ISSUE_8987))  in panfrost_job_get_slot()
146 panfrost_get_job_chain_flag(const struct panfrost_job *job)  in panfrost_get_job_chain_flag()  argument
148 	struct panfrost_fence *f = to_panfrost_fence(job->done_fence);  in panfrost_get_job_chain_flag()
150 	if (!panfrost_has_hw_feature(job->pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION))  in panfrost_get_job_chain_flag()
159 	struct panfrost_job *job = pfdev->jobs[slot][0];  in panfrost_dequeue_job()  local
161 	WARN_ON(!job);  in panfrost_dequeue_job()
163 	if (job->is_profiled && job->engine_usage) {  in panfrost_dequeue_job()
164 		job->engine_usage->elapsed_ns[slot] +=  in panfrost_dequeue_job()
165 			ktime_to_ns(ktime_sub(ktime_get(), job->start_time));  in panfrost_dequeue_job()
166 		job->engine_usage->cycles[slot] +=  in panfrost_dequeue_job()
167 			panfrost_cycle_counter_read(pfdev) - job->start_cycles;  in panfrost_dequeue_job()
170 	if (job->requirements & PANFROST_JD_REQ_CYCLE_COUNT || job->is_profiled)  in panfrost_dequeue_job()
176 	return job;  in panfrost_dequeue_job()
181 		     struct panfrost_job *job)  in panfrost_enqueue_job()  argument
183 	if (WARN_ON(!job))  in panfrost_enqueue_job()
187 		pfdev->jobs[slot][0] = job;  in panfrost_enqueue_job()
192 	pfdev->jobs[slot][1] = job;  in panfrost_enqueue_job()
193 	WARN_ON(panfrost_get_job_chain_flag(job) ==  in panfrost_enqueue_job()
198 static void panfrost_job_hw_submit(struct panfrost_job *job, int js)  in panfrost_job_hw_submit()  argument
200 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_hw_submit()
203 	u64 jc_head = job->jc;  in panfrost_job_hw_submit()
216 	cfg = panfrost_mmu_as_get(pfdev, job->mmu);  in panfrost_job_hw_submit()
221 	panfrost_job_write_affinity(pfdev, job->requirements, js);  in panfrost_job_hw_submit()
228 		panfrost_get_job_chain_flag(job);  in panfrost_job_hw_submit()
239 		job_write(pfdev, JS_FLUSH_ID_NEXT(js), job->flush_id);  in panfrost_job_hw_submit()
244 	subslot = panfrost_enqueue_job(pfdev, js, job);  in panfrost_job_hw_submit()
245 	/* Don't queue the job if a reset is in progress */  in panfrost_job_hw_submit()
247 		job->is_profiled = pfdev->profile_mode;  in panfrost_job_hw_submit()
249 		if (job->requirements & PANFROST_JD_REQ_CYCLE_COUNT ||  in panfrost_job_hw_submit()
250 		    job->is_profiled)  in panfrost_job_hw_submit()
253 		if (job->is_profiled) {  in panfrost_job_hw_submit()
254 			job->start_time = ktime_get();  in panfrost_job_hw_submit()
255 			job->start_cycles = panfrost_cycle_counter_read(pfdev);  in panfrost_job_hw_submit()
261 			job, js, subslot, jc_head, cfg & 0xf);  in panfrost_job_hw_submit()
268 					  struct drm_sched_job *job)  in panfrost_acquire_object_fences()  argument
278 		ret = drm_sched_job_add_implicit_dependencies(job, bos[i],  in panfrost_acquire_object_fences()
297 int panfrost_job_push(struct panfrost_job *job)  in panfrost_job_push()  argument
299 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_push()
303 	ret = drm_gem_lock_reservations(job->bos, job->bo_count,  in panfrost_job_push()
309 	drm_sched_job_arm(&job->base);  in panfrost_job_push()
311 	job->render_done_fence = dma_fence_get(&job->base.s_fence->finished);  in panfrost_job_push()
313 	ret = panfrost_acquire_object_fences(job->bos, job->bo_count,  in panfrost_job_push()
314 					     &job->base);  in panfrost_job_push()
320 	kref_get(&job->refcount); /* put by scheduler job completion */  in panfrost_job_push()
322 	drm_sched_entity_push_job(&job->base);  in panfrost_job_push()
326 	panfrost_attach_object_fences(job->bos, job->bo_count,  in panfrost_job_push()
327 				      job->render_done_fence);  in panfrost_job_push()
330 	drm_gem_unlock_reservations(job->bos, job->bo_count, &acquire_ctx);  in panfrost_job_push()
337 	struct panfrost_job *job = container_of(ref, struct panfrost_job,  in panfrost_job_cleanup()  local
341 	dma_fence_put(job->done_fence);  in panfrost_job_cleanup()
342 	dma_fence_put(job->render_done_fence);  in panfrost_job_cleanup()
344 	if (job->mappings) {  in panfrost_job_cleanup()
345 		for (i = 0; i < job->bo_count; i++) {  in panfrost_job_cleanup()
346 			if (!job->mappings[i])  in panfrost_job_cleanup()
349 			atomic_dec(&job->mappings[i]->obj->gpu_usecount);  in panfrost_job_cleanup()
350 			panfrost_gem_mapping_put(job->mappings[i]);  in panfrost_job_cleanup()
352 		kvfree(job->mappings);  in panfrost_job_cleanup()
355 	if (job->bos) {  in panfrost_job_cleanup()
356 		for (i = 0; i < job->bo_count; i++)  in panfrost_job_cleanup()
357 			drm_gem_object_put(job->bos[i]);  in panfrost_job_cleanup()
359 		kvfree(job->bos);  in panfrost_job_cleanup()
362 	kfree(job);  in panfrost_job_cleanup()
365 void panfrost_job_put(struct panfrost_job *job)  in panfrost_job_put()  argument
367 	kref_put(&job->refcount, panfrost_job_cleanup);  in panfrost_job_put()
372 	struct panfrost_job *job = to_panfrost_job(sched_job);  in panfrost_job_free()  local
376 	panfrost_job_put(job);  in panfrost_job_free()
381 	struct panfrost_job *job = to_panfrost_job(sched_job);  in panfrost_job_run()  local
382 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_run()
383 	int slot = panfrost_job_get_slot(job);  in panfrost_job_run()
386 	if (unlikely(job->base.s_fence->finished.error))  in panfrost_job_run()
389 	/* Nothing to execute: can happen if the job has finished while  in panfrost_job_run()
392 	if (!job->jc)  in panfrost_job_run()
399 	if (job->done_fence)  in panfrost_job_run()
400 		dma_fence_put(job->done_fence);  in panfrost_job_run()
401 	job->done_fence = dma_fence_get(fence);  in panfrost_job_run()
403 	panfrost_job_hw_submit(job, slot);  in panfrost_job_run()
432 				    struct panfrost_job *job,  in panfrost_job_handle_err()  argument
452 		/* Update the job head so we can resume */  in panfrost_job_handle_err()
453 		job->jc = job_read(pfdev, JS_TAIL_LO(js)) |  in panfrost_job_handle_err()
456 		/* The job will be resumed, don't signal the fence */  in panfrost_job_handle_err()
459 		/* Job has been hard-stopped, flag it as canceled */  in panfrost_job_handle_err()
460 		dma_fence_set_error(job->done_fence, -ECANCELED);  in panfrost_job_handle_err()
461 		job->jc = 0;  in panfrost_job_handle_err()
467 		dma_fence_set_error(job->done_fence, -EINVAL);  in panfrost_job_handle_err()
468 		job->jc = 0;  in panfrost_job_handle_err()
471 	panfrost_mmu_as_put(pfdev, job->mmu);  in panfrost_job_handle_err()
475 		dma_fence_signal_locked(job->done_fence);  in panfrost_job_handle_err()
486 				     struct panfrost_job *job)  in panfrost_job_handle_done()  argument
488 	/* Set ->jc to 0 to avoid re-submitting an already finished job (can  in panfrost_job_handle_done()
491 	job->jc = 0;  in panfrost_job_handle_done()
492 	panfrost_mmu_as_put(pfdev, job->mmu);  in panfrost_job_handle_done()
495 	dma_fence_signal_locked(job->done_fence);  in panfrost_job_handle_done()
535 		 * is racy. If we only have one job done at the time we  in panfrost_job_handle_irq()
536 		 * read JOB_INT_RAWSTAT but the second job fails before we  in panfrost_job_handle_irq()
558 			/* When the current job doesn't fail, the JM dequeues  in panfrost_job_handle_irq()
559 			 * the next job without waiting for an ACK, this means  in panfrost_job_handle_irq()
562 			 * are inactive, but one job remains in pfdev->jobs[j],  in panfrost_job_handle_irq()
565 			 * job in _NEXT (see above).  in panfrost_job_handle_irq()
588 			/* The job was cancelled, signal the fence now */  in panfrost_job_handle_irq()
594 			/* Requeue the job we removed if no reset is pending */  in panfrost_job_handle_irq()
651 	 * to release job resources. We should rework the code to follow this  in panfrost_reset()
668 	/* Mask job interrupts and synchronize to make sure we won't be  in panfrost_reset()
675 		/* Cancel the next job and soft-stop the running job. */  in panfrost_reset()
694 	 * panfrost_devfreq_record_idle() for each stuck job.  in panfrost_reset()
714 	/* panfrost_device_reset() unmasks job interrupts, but we want to  in panfrost_reset()
725 	 * while resubmitting jobs because the job submission logic will  in panfrost_reset()
738 	/* Re-enable job interrupts now that everything has been restarted. */  in panfrost_reset()
749 	struct panfrost_job *job = to_panfrost_job(sched_job);  in panfrost_job_timedout()  local
750 	struct panfrost_device *pfdev = job->pfdev;  in panfrost_job_timedout()
751 	int js = panfrost_job_get_slot(job);  in panfrost_job_timedout()
757 	if (dma_fence_is_signaled(job->done_fence))  in panfrost_job_timedout()
766 	 * job timeouts, synchronize the IRQ handler and re-check the fence  in panfrost_job_timedout()
771 	if (dma_fence_is_signaled(job->done_fence)) {  in panfrost_job_timedout()
784 	panfrost_core_dump(job);  in panfrost_job_timedout()
844 	 * disambiguation stopping the right job in the close path is tricky,  in panfrost_job_init()
857 	js->irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "job");  in panfrost_job_init()
864 					IRQF_SHARED, KBUILD_MODNAME "-job",  in panfrost_job_init()
867 		dev_err(pfdev->dev, "failed to request job irq");  in panfrost_job_init()
951 			struct panfrost_job *job = pfdev->jobs[i][j];  in panfrost_job_close()  local
954 			if (!job || job->base.entity != entity)  in panfrost_job_close()
958 				/* Try to cancel the job before it starts */  in panfrost_job_close()
960 				/* Reset the job head so it doesn't get restarted if  in panfrost_job_close()
961 				 * the job in the first slot failed.  in panfrost_job_close()
963 				job->jc = 0;  in panfrost_job_close()
967 				cmd = panfrost_get_job_chain_flag(job) ?  in panfrost_job_close()
977 			job->engine_usage = NULL;  in panfrost_job_close()