27 static void ivpu_cmdq_ring_db(struct ivpu_device *vdev, struct ivpu_cmdq *cmdq)  in ivpu_cmdq_ring_db()  argument
29 	ivpu_hw_db_set(vdev, cmdq->db_id);  in ivpu_cmdq_ring_db()
33 					  struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)  in ivpu_preemption_buffers_create()  argument
42 	cmdq->primary_preempt_buf = ivpu_bo_create(vdev, &file_priv->ctx, &vdev->hw->ranges.user,  in ivpu_preemption_buffers_create()
44 	if (!cmdq->primary_preempt_buf) {  in ivpu_preemption_buffers_create()
49 	cmdq->secondary_preempt_buf = ivpu_bo_create(vdev, &file_priv->ctx, &vdev->hw->ranges.dma,  in ivpu_preemption_buffers_create()
51 	if (!cmdq->secondary_preempt_buf) {  in ivpu_preemption_buffers_create()
59 	ivpu_bo_free(cmdq->primary_preempt_buf);  in ivpu_preemption_buffers_create()
60 	cmdq->primary_preempt_buf = NULL;  in ivpu_preemption_buffers_create()
65 					 struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)  in ivpu_preemption_buffers_free()  argument
70 	if (cmdq->primary_preempt_buf)  in ivpu_preemption_buffers_free()
71 		ivpu_bo_free(cmdq->primary_preempt_buf);  in ivpu_preemption_buffers_free()
72 	if (cmdq->secondary_preempt_buf)  in ivpu_preemption_buffers_free()
73 		ivpu_bo_free(cmdq->secondary_preempt_buf);  in ivpu_preemption_buffers_free()
79 	struct ivpu_cmdq *cmdq;  in ivpu_cmdq_alloc()  local
82 	cmdq = kzalloc(sizeof(*cmdq), GFP_KERNEL);  in ivpu_cmdq_alloc()
83 	if (!cmdq)  in ivpu_cmdq_alloc()
86 	ret = xa_alloc_cyclic(&vdev->db_xa, &cmdq->db_id, NULL, vdev->db_limit, &vdev->db_next,  in ivpu_cmdq_alloc()
93 	ret = xa_alloc_cyclic(&file_priv->cmdq_xa, &cmdq->id, cmdq, file_priv->cmdq_limit,  in ivpu_cmdq_alloc()
100 	cmdq->mem = ivpu_bo_create_global(vdev, SZ_4K, DRM_IVPU_BO_WC | DRM_IVPU_BO_MAPPABLE);  in ivpu_cmdq_alloc()
101 	if (!cmdq->mem)  in ivpu_cmdq_alloc()
104 	ret = ivpu_preemption_buffers_create(vdev, file_priv, cmdq);  in ivpu_cmdq_alloc()
108 	return cmdq;  in ivpu_cmdq_alloc()
111 	xa_erase(&file_priv->cmdq_xa, cmdq->id);  in ivpu_cmdq_alloc()
113 	xa_erase(&vdev->db_xa, cmdq->db_id);  in ivpu_cmdq_alloc()
115 	kfree(cmdq);  in ivpu_cmdq_alloc()
119 static void ivpu_cmdq_free(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)  in ivpu_cmdq_free()  argument
121 	if (!cmdq)  in ivpu_cmdq_free()
124 	ivpu_preemption_buffers_free(file_priv->vdev, file_priv, cmdq);  in ivpu_cmdq_free()
125 	ivpu_bo_free(cmdq->mem);  in ivpu_cmdq_free()
126 	xa_erase(&file_priv->vdev->db_xa, cmdq->db_id);  in ivpu_cmdq_free()
127 	kfree(cmdq);  in ivpu_cmdq_free()
130 static int ivpu_hws_cmdq_init(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq, u16 engine,  in ivpu_hws_cmdq_init()  argument
136 	ret = ivpu_jsm_hws_create_cmdq(vdev, file_priv->ctx.id, file_priv->ctx.id, cmdq->id,  in ivpu_hws_cmdq_init()
138 				       cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem));  in ivpu_hws_cmdq_init()
142 	ret = ivpu_jsm_hws_set_context_sched_properties(vdev, file_priv->ctx.id, cmdq->id,  in ivpu_hws_cmdq_init()
150 static int ivpu_register_db(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)  in ivpu_register_db()  argument
156 		ret = ivpu_jsm_hws_register_db(vdev, file_priv->ctx.id, cmdq->id, cmdq->db_id,  in ivpu_register_db()
157 					       cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem));  in ivpu_register_db()
159 		ret = ivpu_jsm_register_db(vdev, file_priv->ctx.id, cmdq->db_id,  in ivpu_register_db()
160 					   cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem));  in ivpu_register_db()
163 		ivpu_dbg(vdev, JOB, "DB %d registered to cmdq %d ctx %d\n",  in ivpu_register_db()
164 			 cmdq->db_id, cmdq->id, file_priv->ctx.id);  in ivpu_register_db()
170 ivpu_cmdq_init(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq, u8 priority)  in ivpu_cmdq_init()  argument
178 	if (cmdq->db_registered)  in ivpu_cmdq_init()
181 	cmdq->entry_count = (u32)((ivpu_bo_size(cmdq->mem) - sizeof(struct vpu_job_queue_header)) /  in ivpu_cmdq_init()
184 	cmdq->jobq = (struct vpu_job_queue *)ivpu_bo_vaddr(cmdq->mem);  in ivpu_cmdq_init()
185 	jobq_header = &cmdq->jobq->header;  in ivpu_cmdq_init()
197 		ret = ivpu_hws_cmdq_init(file_priv, cmdq, VPU_ENGINE_COMPUTE, priority);  in ivpu_cmdq_init()
202 	ret = ivpu_register_db(file_priv, cmdq);  in ivpu_cmdq_init()
206 	cmdq->db_registered = true;  in ivpu_cmdq_init()
211 static int ivpu_cmdq_fini(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)  in ivpu_cmdq_fini()  argument
218 	if (!cmdq->db_registered)  in ivpu_cmdq_fini()
221 	cmdq->db_registered = false;  in ivpu_cmdq_fini()
224 		ret = ivpu_jsm_hws_destroy_cmdq(vdev, file_priv->ctx.id, cmdq->id);  in ivpu_cmdq_fini()
226 			ivpu_dbg(vdev, JOB, "Command queue %d destroyed\n", cmdq->id);  in ivpu_cmdq_fini()
229 	ret = ivpu_jsm_unregister_db(vdev, cmdq->db_id);  in ivpu_cmdq_fini()
231 		ivpu_dbg(vdev, JOB, "DB %d unregistered\n", cmdq->db_id);  in ivpu_cmdq_fini()
238 	struct ivpu_cmdq *cmdq;  in ivpu_cmdq_acquire()  local
244 	xa_for_each(&file_priv->cmdq_xa, cmdq_id, cmdq)  in ivpu_cmdq_acquire()
245 		if (cmdq->priority == priority)  in ivpu_cmdq_acquire()
248 	if (!cmdq) {  in ivpu_cmdq_acquire()
249 		cmdq = ivpu_cmdq_alloc(file_priv);  in ivpu_cmdq_acquire()
250 		if (!cmdq)  in ivpu_cmdq_acquire()
252 		cmdq->priority = priority;  in ivpu_cmdq_acquire()
255 	ret = ivpu_cmdq_init(file_priv, cmdq, priority);  in ivpu_cmdq_acquire()
259 	return cmdq;  in ivpu_cmdq_acquire()
264 	struct ivpu_cmdq *cmdq;  in ivpu_cmdq_release_all_locked()  local
269 	xa_for_each(&file_priv->cmdq_xa, cmdq_id, cmdq) {  in ivpu_cmdq_release_all_locked()
271 		ivpu_cmdq_fini(file_priv, cmdq);  in ivpu_cmdq_release_all_locked()
272 		ivpu_cmdq_free(file_priv, cmdq);  in ivpu_cmdq_release_all_locked()
284 	struct ivpu_cmdq *cmdq;  in ivpu_cmdq_reset()  local
289 	xa_for_each(&file_priv->cmdq_xa, cmdq_id, cmdq)  in ivpu_cmdq_reset()
290 		cmdq->db_registered = false;  in ivpu_cmdq_reset()
310 	struct ivpu_cmdq *cmdq;  in ivpu_cmdq_fini_all()  local
313 	xa_for_each(&file_priv->cmdq_xa, cmdq_id, cmdq)  in ivpu_cmdq_fini_all()
314 		ivpu_cmdq_fini(file_priv, cmdq);  in ivpu_cmdq_fini_all()
329 static int ivpu_cmdq_push_job(struct ivpu_cmdq *cmdq, struct ivpu_job *job)  in ivpu_cmdq_push_job()  argument
332 	struct vpu_job_queue_header *header = &cmdq->jobq->header;  in ivpu_cmdq_push_job()
335 	u32 next_entry = (tail + 1) % cmdq->entry_count;  in ivpu_cmdq_push_job()
339 		ivpu_dbg(vdev, JOB, "Job queue full: ctx %d cmdq %d db %d head %d tail %d\n",  in ivpu_cmdq_push_job()
340 			 job->file_priv->ctx.id, cmdq->id, cmdq->db_id, header->head, tail);  in ivpu_cmdq_push_job()
344 	entry = &cmdq->jobq->slot[tail].job;  in ivpu_cmdq_push_job()
352 		if (cmdq->primary_preempt_buf) {  in ivpu_cmdq_push_job()
353 			entry->primary_preempt_buf_addr = cmdq->primary_preempt_buf->vpu_addr;  in ivpu_cmdq_push_job()
354 			entry->primary_preempt_buf_size = ivpu_bo_size(cmdq->primary_preempt_buf);  in ivpu_cmdq_push_job()
357 		if (cmdq->secondary_preempt_buf) {  in ivpu_cmdq_push_job()
358 			entry->secondary_preempt_buf_addr = cmdq->secondary_preempt_buf->vpu_addr;  in ivpu_cmdq_push_job()
360 				ivpu_bo_size(cmdq->secondary_preempt_buf);  in ivpu_cmdq_push_job()
516 	struct ivpu_cmdq *cmdq;  in ivpu_job_submit()  local
526 	cmdq = ivpu_cmdq_acquire(file_priv, priority);  in ivpu_job_submit()
527 	if (!cmdq) {  in ivpu_job_submit()
545 	ret = ivpu_cmdq_push_job(cmdq, job);  in ivpu_job_submit()
552 		cmdq->jobq->header.head = cmdq->jobq->header.tail;  in ivpu_job_submit()
555 		ivpu_cmdq_ring_db(vdev, cmdq);  in ivpu_job_submit()
563 		 job->cmd_buf_vpu_addr, cmdq->jobq->header.tail);  in ivpu_job_submit()