163 		ivpu_dbg(vdev, JOB, "DB %d registered to cmdq %d ctx %d\n",  in ivpu_register_db()
190 		ivpu_dbg(vdev, JOB, "Turbo mode enabled");  in ivpu_cmdq_init()
226 			ivpu_dbg(vdev, JOB, "Command queue %d destroyed\n", cmdq->id);  in ivpu_cmdq_fini()
231 		ivpu_dbg(vdev, JOB, "DB %d unregistered\n", cmdq->db_id);  in ivpu_cmdq_fini()
279  * and FW loses job queue state. The next time job queue is used it
329 static int ivpu_cmdq_push_job(struct ivpu_cmdq *cmdq, struct ivpu_job *job)  in ivpu_cmdq_push_job()  argument
331 	struct ivpu_device *vdev = job->vdev;  in ivpu_cmdq_push_job()
337 	/* Check if there is space left in job queue */  in ivpu_cmdq_push_job()
339 		ivpu_dbg(vdev, JOB, "Job queue full: ctx %d cmdq %d db %d head %d tail %d\n",  in ivpu_cmdq_push_job()
340 			 job->file_priv->ctx.id, cmdq->id, cmdq->db_id, header->head, tail);  in ivpu_cmdq_push_job()
344 	entry = &cmdq->jobq->slot[tail].job;  in ivpu_cmdq_push_job()
345 	entry->batch_buf_addr = job->cmd_buf_vpu_addr;  in ivpu_cmdq_push_job()
346 	entry->job_id = job->job_id;  in ivpu_cmdq_push_job()
414 static void ivpu_job_destroy(struct ivpu_job *job)  in ivpu_job_destroy()  argument
416 	struct ivpu_device *vdev = job->vdev;  in ivpu_job_destroy()
419 	ivpu_dbg(vdev, JOB, "Job destroyed: id %3u ctx %2d engine %d",  in ivpu_job_destroy()
420 		 job->job_id, job->file_priv->ctx.id, job->engine_idx);  in ivpu_job_destroy()
422 	for (i = 0; i < job->bo_count; i++)  in ivpu_job_destroy()
423 		if (job->bos[i])  in ivpu_job_destroy()
424 			drm_gem_object_put(&job->bos[i]->base.base);  in ivpu_job_destroy()
426 	dma_fence_put(job->done_fence);  in ivpu_job_destroy()
427 	ivpu_file_priv_put(&job->file_priv);  in ivpu_job_destroy()
428 	kfree(job);  in ivpu_job_destroy()
435 	struct ivpu_job *job;  in ivpu_job_create()  local
437 	job = kzalloc(struct_size(job, bos, bo_count), GFP_KERNEL);  in ivpu_job_create()
438 	if (!job)  in ivpu_job_create()
441 	job->vdev = vdev;  in ivpu_job_create()
442 	job->engine_idx = engine_idx;  in ivpu_job_create()
443 	job->bo_count = bo_count;  in ivpu_job_create()
444 	job->done_fence = ivpu_fence_create(vdev);  in ivpu_job_create()
445 	if (!job->done_fence) {  in ivpu_job_create()
450 	job->file_priv = ivpu_file_priv_get(file_priv);  in ivpu_job_create()
452 	trace_job("create", job);  in ivpu_job_create()
453 	ivpu_dbg(vdev, JOB, "Job created: ctx %2d engine %d", file_priv->ctx.id, job->engine_idx);  in ivpu_job_create()
454 	return job;  in ivpu_job_create()
457 	kfree(job);  in ivpu_job_create()
463 	struct ivpu_job *job;  in ivpu_job_remove_from_submitted_jobs()  local
466 	job = __xa_erase(&vdev->submitted_jobs_xa, job_id);  in ivpu_job_remove_from_submitted_jobs()
468 	if (xa_empty(&vdev->submitted_jobs_xa) && job) {  in ivpu_job_remove_from_submitted_jobs()
475 	return job;  in ivpu_job_remove_from_submitted_jobs()
480 	struct ivpu_job *job;  in ivpu_job_signal_and_destroy()  local
482 	job = ivpu_job_remove_from_submitted_jobs(vdev, job_id);  in ivpu_job_signal_and_destroy()
483 	if (!job)  in ivpu_job_signal_and_destroy()
486 	if (job->file_priv->has_mmu_faults)  in ivpu_job_signal_and_destroy()
489 	job->bos[CMD_BUF_IDX]->job_status = job_status;  in ivpu_job_signal_and_destroy()
490 	dma_fence_signal(job->done_fence);  in ivpu_job_signal_and_destroy()
492 	trace_job("done", job);  in ivpu_job_signal_and_destroy()
493 	ivpu_dbg(vdev, JOB, "Job complete:  id %3u ctx %2d engine %d status 0x%x\n",  in ivpu_job_signal_and_destroy()
494 		 job->job_id, job->file_priv->ctx.id, job->engine_idx, job_status);  in ivpu_job_signal_and_destroy()
496 	ivpu_job_destroy(job);  in ivpu_job_signal_and_destroy()
505 	struct ivpu_job *job;  in ivpu_jobs_abort_all()  local
508 	xa_for_each(&vdev->submitted_jobs_xa, id, job)  in ivpu_jobs_abort_all()
512 static int ivpu_job_submit(struct ivpu_job *job, u8 priority)  in ivpu_job_submit()  argument
514 	struct ivpu_file_priv *file_priv = job->file_priv;  in ivpu_job_submit()
515 	struct ivpu_device *vdev = job->vdev;  in ivpu_job_submit()
528 		ivpu_warn_ratelimited(vdev, "Failed to get job queue, ctx %d engine %d prio %d\n",  in ivpu_job_submit()
529 				      file_priv->ctx.id, job->engine_idx, priority);  in ivpu_job_submit()
536 	ret = __xa_alloc_cyclic(&vdev->submitted_jobs_xa, &job->job_id, job, file_priv->job_limit,  in ivpu_job_submit()
539 		ivpu_dbg(vdev, JOB, "Too many active jobs in ctx %d\n",  in ivpu_job_submit()
545 	ret = ivpu_cmdq_push_job(cmdq, job);  in ivpu_job_submit()
560 	trace_job("submit", job);  in ivpu_job_submit()
561 	ivpu_dbg(vdev, JOB, "Job submitted: id %3u ctx %2d engine %d prio %d addr 0x%llx next %d\n",  in ivpu_job_submit()
562 		 job->job_id, file_priv->ctx.id, job->engine_idx, priority,  in ivpu_job_submit()
563 		 job->cmd_buf_vpu_addr, cmdq->jobq->header.tail);  in ivpu_job_submit()
570 		ivpu_job_signal_and_destroy(vdev, job->job_id, VPU_JSM_STATUS_SUCCESS);  in ivpu_job_submit()
575 	__xa_erase(&vdev->submitted_jobs_xa, job->job_id);  in ivpu_job_submit()
585 ivpu_job_prepare_bos_for_submit(struct drm_file *file, struct ivpu_job *job, u32 *buf_handles,  in ivpu_job_prepare_bos_for_submit()  argument
602 		job->bos[i] = to_ivpu_bo(obj);  in ivpu_job_prepare_bos_for_submit()
604 		ret = ivpu_bo_pin(job->bos[i]);  in ivpu_job_prepare_bos_for_submit()
609 	bo = job->bos[CMD_BUF_IDX];  in ivpu_job_prepare_bos_for_submit()
620 	job->cmd_buf_vpu_addr = bo->vpu_addr + commands_offset;  in ivpu_job_prepare_bos_for_submit()
622 	ret = drm_gem_lock_reservations((struct drm_gem_object **)job->bos, buf_count,  in ivpu_job_prepare_bos_for_submit()
630 		ret = dma_resv_reserve_fences(job->bos[i]->base.base.resv, 1);  in ivpu_job_prepare_bos_for_submit()
639 		dma_resv_add_fence(job->bos[i]->base.base.resv, job->done_fence, usage);  in ivpu_job_prepare_bos_for_submit()
643 	drm_gem_unlock_reservations((struct drm_gem_object **)job->bos, buf_count, &acquire_ctx);  in ivpu_job_prepare_bos_for_submit()
663 	struct ivpu_job *job;  in ivpu_submit_ioctl()  local
703 	ivpu_dbg(vdev, JOB, "Submit ioctl: ctx %u buf_count %u\n",  in ivpu_submit_ioctl()
706 	job = ivpu_job_create(file_priv, params->engine, params->buffer_count);  in ivpu_submit_ioctl()
707 	if (!job) {  in ivpu_submit_ioctl()
708 		ivpu_err(vdev, "Failed to create job\n");  in ivpu_submit_ioctl()
713 	ret = ivpu_job_prepare_bos_for_submit(file, job, buf_handles, params->buffer_count,  in ivpu_submit_ioctl()
716 		ivpu_err(vdev, "Failed to prepare job: %d\n", ret);  in ivpu_submit_ioctl()
723 	ret = ivpu_job_submit(job, priority);  in ivpu_submit_ioctl()
733 	dma_fence_signal(job->done_fence);  in ivpu_submit_ioctl()
735 	ivpu_job_destroy(job);  in ivpu_submit_ioctl()