14  * we can attach fences and update the reservations after pushing the job
22 v3d_lock_bo_reservations(struct v3d_job *job,  in v3d_lock_bo_reservations()  argument
27 	ret = drm_gem_lock_reservations(job->bo, job->bo_count, acquire_ctx);  in v3d_lock_bo_reservations()
31 	for (i = 0; i < job->bo_count; i++) {  in v3d_lock_bo_reservations()
32 		ret = dma_resv_reserve_fences(job->bo[i]->resv, 1);  in v3d_lock_bo_reservations()
36 		ret = drm_sched_job_add_implicit_dependencies(&job->base,  in v3d_lock_bo_reservations()
37 							      job->bo[i], true);  in v3d_lock_bo_reservations()
45 	drm_gem_unlock_reservations(job->bo, job->bo_count, acquire_ctx);  in v3d_lock_bo_reservations()
50  * v3d_lookup_bos() - Sets up job->bo[] with the GEM objects
51  * referenced by the job.
54  * @job: V3D job being set up
59  * the submitted job's BO list. This does the validation of the job's
60  * BO list and reference counting for the lifetime of the job.
68 	       struct v3d_job *job,  in v3d_lookup_bos()  argument
72 	job->bo_count = bo_count;  in v3d_lookup_bos()
74 	if (!job->bo_count) {  in v3d_lookup_bos()
84 				      job->bo_count, &job->bo);  in v3d_lookup_bos()
90 	struct v3d_job *job = container_of(ref, struct v3d_job, refcount);  in v3d_job_free()  local
93 	if (job->bo) {  in v3d_job_free()
94 		for (i = 0; i < job->bo_count; i++)  in v3d_job_free()
95 			drm_gem_object_put(job->bo[i]);  in v3d_job_free()
96 		kvfree(job->bo);  in v3d_job_free()
99 	dma_fence_put(job->irq_fence);  in v3d_job_free()
100 	dma_fence_put(job->done_fence);  in v3d_job_free()
102 	if (job->perfmon)  in v3d_job_free()
103 		v3d_perfmon_put(job->perfmon);  in v3d_job_free()
105 	kfree(job);  in v3d_job_free()
111 	struct v3d_render_job *job = container_of(ref, struct v3d_render_job,  in v3d_render_job_free()  local
115 	list_for_each_entry_safe(bo, save, &job->unref_list, unref_head) {  in v3d_render_job_free()
122 void v3d_job_cleanup(struct v3d_job *job)  in v3d_job_cleanup()  argument
124 	if (!job)  in v3d_job_cleanup()
127 	drm_sched_job_cleanup(&job->base);  in v3d_job_cleanup()
128 	v3d_job_put(job);  in v3d_job_cleanup()
131 void v3d_job_put(struct v3d_job *job)  in v3d_job_put()  argument
133 	if (!job)  in v3d_job_put()
136 	kref_put(&job->refcount, job->free);  in v3d_job_put()
144 		DRM_ERROR("Cannot allocate memory for V3D job.\n");  in v3d_job_allocate()
160 	     struct v3d_job *job, void (*free)(struct kref *ref),  in v3d_job_init()  argument
167 	job->v3d = v3d;  in v3d_job_init()
168 	job->free = free;  in v3d_job_init()
169 	job->file = file_priv;  in v3d_job_init()
171 	ret = drm_sched_job_init(&job->base, &v3d_priv->sched_entity[queue],  in v3d_job_init()
188 				ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, in.handle, 0);  in v3d_job_init()
196 		ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, in_sync, 0);  in v3d_job_init()
203 	kref_init(&job->refcount);  in v3d_job_init()
208 	drm_sched_job_cleanup(&job->base);  in v3d_job_init()
213 v3d_push_job(struct v3d_job *job)  in v3d_push_job()  argument
215 	drm_sched_job_arm(&job->base);  in v3d_push_job()
217 	job->done_fence = dma_fence_get(&job->base.s_fence->finished);  in v3d_push_job()
219 	/* put by scheduler job completion */  in v3d_push_job()
220 	kref_get(&job->refcount);  in v3d_push_job()
222 	drm_sched_entity_push_job(&job->base);  in v3d_push_job()
227 					 struct v3d_job *job,  in v3d_attach_fences_and_unlock_reservation()  argument
237 	for (i = 0; i < job->bo_count; i++) {  in v3d_attach_fences_and_unlock_reservation()
239 		dma_resv_add_fence(job->bo[i]->resv, job->done_fence,  in v3d_attach_fences_and_unlock_reservation()
243 	drm_gem_unlock_reservations(job->bo, job->bo_count, acquire_ctx);  in v3d_attach_fences_and_unlock_reservation()
245 	/* Update the return sync object for the job */  in v3d_attach_fences_and_unlock_reservation()
271 			   struct v3d_csd_job **job,  in v3d_setup_csd_jobs_and_bos()  argument
278 	ret = v3d_job_allocate((void *)job, sizeof(**job));  in v3d_setup_csd_jobs_and_bos()
282 	ret = v3d_job_init(v3d, file_priv, &(*job)->base,  in v3d_setup_csd_jobs_and_bos()
285 		v3d_job_deallocate((void *)job);  in v3d_setup_csd_jobs_and_bos()
300 	(*job)->args = *args;  in v3d_setup_csd_jobs_and_bos()
372  * to be signaled when job completes (out_sync).
383 		DRM_DEBUG("Two multisync extensions were added to the same job.");  in v3d_get_multisync_submit_deps()
406 /* Get data for the indirect CSD job submission. */
410 				struct v3d_cpu_job *job)  in v3d_get_cpu_indirect_csd_params()  argument
415 	struct v3d_indirect_csd_info *info = &job->indirect_csd;  in v3d_get_cpu_indirect_csd_params()
417 	if (!job) {  in v3d_get_cpu_indirect_csd_params()
418 		DRM_DEBUG("CPU job extension was attached to a GPU job.\n");  in v3d_get_cpu_indirect_csd_params()
422 	if (job->job_type) {  in v3d_get_cpu_indirect_csd_params()
423 		DRM_DEBUG("Two CPU job extensions were added to the same CPU job.\n");  in v3d_get_cpu_indirect_csd_params()
435 	job->job_type = V3D_CPU_JOB_TYPE_INDIRECT_CSD;  in v3d_get_cpu_indirect_csd_params()
444 					  &info->job, &info->clean_job,  in v3d_get_cpu_indirect_csd_params()
448 /* Get data for the query timestamp job submission. */
452 				   struct v3d_cpu_job *job)  in v3d_get_cpu_timestamp_query_params()  argument
456 	struct v3d_timestamp_query_info *query_info = &job->timestamp_query;  in v3d_get_cpu_timestamp_query_params()
460 	if (!job) {  in v3d_get_cpu_timestamp_query_params()
461 		DRM_DEBUG("CPU job extension was attached to a GPU job.\n");  in v3d_get_cpu_timestamp_query_params()
465 	if (job->job_type) {  in v3d_get_cpu_timestamp_query_params()
466 		DRM_DEBUG("Two CPU job extensions were added to the same CPU job.\n");  in v3d_get_cpu_timestamp_query_params()
476 	job->job_type = V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY;  in v3d_get_cpu_timestamp_query_params()
514 	v3d_timestamp_query_info_free(&job->timestamp_query, i);  in v3d_get_cpu_timestamp_query_params()
521 				   struct v3d_cpu_job *job)  in v3d_get_cpu_reset_timestamp_params()  argument
525 	struct v3d_timestamp_query_info *query_info = &job->timestamp_query;  in v3d_get_cpu_reset_timestamp_params()
529 	if (!job) {  in v3d_get_cpu_reset_timestamp_params()
530 		DRM_DEBUG("CPU job extension was attached to a GPU job.\n");  in v3d_get_cpu_reset_timestamp_params()
534 	if (job->job_type) {  in v3d_get_cpu_reset_timestamp_params()
535 		DRM_DEBUG("Two CPU job extensions were added to the same CPU job.\n");  in v3d_get_cpu_reset_timestamp_params()
542 	job->job_type = V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY;  in v3d_get_cpu_reset_timestamp_params()
574 	v3d_timestamp_query_info_free(&job->timestamp_query, i);  in v3d_get_cpu_reset_timestamp_params()
578 /* Get data for the copy timestamp query results job submission. */
582 				      struct v3d_cpu_job *job)  in v3d_get_cpu_copy_query_results_params()  argument
586 	struct v3d_timestamp_query_info *query_info = &job->timestamp_query;  in v3d_get_cpu_copy_query_results_params()
590 	if (!job) {  in v3d_get_cpu_copy_query_results_params()
591 		DRM_DEBUG("CPU job extension was attached to a GPU job.\n");  in v3d_get_cpu_copy_query_results_params()
595 	if (job->job_type) {  in v3d_get_cpu_copy_query_results_params()
596 		DRM_DEBUG("Two CPU job extensions were added to the same CPU job.\n");  in v3d_get_cpu_copy_query_results_params()
606 	job->job_type = V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY;  in v3d_get_cpu_copy_query_results_params()
641 	job->copy.do_64bit = copy.do_64bit;  in v3d_get_cpu_copy_query_results_params()
642 	job->copy.do_partial = copy.do_partial;  in v3d_get_cpu_copy_query_results_params()
643 	job->copy.availability_bit = copy.availability_bit;  in v3d_get_cpu_copy_query_results_params()
644 	job->copy.offset = copy.offset;  in v3d_get_cpu_copy_query_results_params()
645 	job->copy.stride = copy.stride;  in v3d_get_cpu_copy_query_results_params()
650 	v3d_timestamp_query_info_free(&job->timestamp_query, i);  in v3d_get_cpu_copy_query_results_params()
720 				     struct v3d_cpu_job *job)  in v3d_get_cpu_reset_performance_params()  argument
722 	struct v3d_performance_query_info *query_info = &job->performance_query;  in v3d_get_cpu_reset_performance_params()
726 	if (!job) {  in v3d_get_cpu_reset_performance_params()
727 		DRM_DEBUG("CPU job extension was attached to a GPU job.\n");  in v3d_get_cpu_reset_performance_params()
731 	if (job->job_type) {  in v3d_get_cpu_reset_performance_params()
732 		DRM_DEBUG("Two CPU job extensions were added to the same CPU job.\n");  in v3d_get_cpu_reset_performance_params()
739 	job->job_type = V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY;  in v3d_get_cpu_reset_performance_params()
766 					  struct v3d_cpu_job *job)  in v3d_get_cpu_copy_performance_query_params()  argument
768 	struct v3d_performance_query_info *query_info = &job->performance_query;  in v3d_get_cpu_copy_performance_query_params()
772 	if (!job) {  in v3d_get_cpu_copy_performance_query_params()
773 		DRM_DEBUG("CPU job extension was attached to a GPU job.\n");  in v3d_get_cpu_copy_performance_query_params()
777 	if (job->job_type) {  in v3d_get_cpu_copy_performance_query_params()
778 		DRM_DEBUG("Two CPU job extensions were added to the same CPU job.\n");  in v3d_get_cpu_copy_performance_query_params()
788 	job->job_type = V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY;  in v3d_get_cpu_copy_performance_query_params()
810 	job->copy.do_64bit = copy.do_64bit;  in v3d_get_cpu_copy_performance_query_params()
811 	job->copy.do_partial = copy.do_partial;  in v3d_get_cpu_copy_performance_query_params()
812 	job->copy.availability_bit = copy.availability_bit;  in v3d_get_cpu_copy_performance_query_params()
813 	job->copy.offset = copy.offset;  in v3d_get_cpu_copy_performance_query_params()
814 	job->copy.stride = copy.stride;  in v3d_get_cpu_copy_performance_query_params()
826 		   struct v3d_cpu_job *job)  in v3d_get_extensions()  argument
845 			ret = v3d_get_cpu_indirect_csd_params(file_priv, user_ext, job);  in v3d_get_extensions()
848 			ret = v3d_get_cpu_timestamp_query_params(file_priv, user_ext, job);  in v3d_get_extensions()
851 			ret = v3d_get_cpu_reset_timestamp_params(file_priv, user_ext, job);  in v3d_get_extensions()
854 			ret = v3d_get_cpu_copy_query_results_params(file_priv, user_ext, job);  in v3d_get_extensions()
857 			ret = v3d_get_cpu_reset_performance_params(file_priv, user_ext, job);  in v3d_get_extensions()
860 			ret = v3d_get_cpu_copy_performance_query_params(file_priv, user_ext, job);  in v3d_get_extensions()
877  * v3d_submit_cl_ioctl() - Submits a job (frame) to the V3D.
1055  * v3d_submit_tfu_ioctl() - Submits a TFU (texture formatting) job to the V3D.
1070 	struct v3d_tfu_job *job = NULL;  in v3d_submit_tfu_ioctl()  local
1089 	ret = v3d_job_allocate((void *)&job, sizeof(*job));  in v3d_submit_tfu_ioctl()
1093 	ret = v3d_job_init(v3d, file_priv, &job->base,  in v3d_submit_tfu_ioctl()
1096 		v3d_job_deallocate((void *)&job);  in v3d_submit_tfu_ioctl()
1100 	job->base.bo = kcalloc(ARRAY_SIZE(args->bo_handles),  in v3d_submit_tfu_ioctl()
1101 			       sizeof(*job->base.bo), GFP_KERNEL);  in v3d_submit_tfu_ioctl()
1102 	if (!job->base.bo) {  in v3d_submit_tfu_ioctl()
1107 	job->args = *args;  in v3d_submit_tfu_ioctl()
1109 	for (job->base.bo_count = 0;  in v3d_submit_tfu_ioctl()
1110 	     job->base.bo_count < ARRAY_SIZE(args->bo_handles);  in v3d_submit_tfu_ioctl()
1111 	     job->base.bo_count++) {  in v3d_submit_tfu_ioctl()
1114 		if (!args->bo_handles[job->base.bo_count])  in v3d_submit_tfu_ioctl()
1117 		bo = drm_gem_object_lookup(file_priv, args->bo_handles[job->base.bo_count]);  in v3d_submit_tfu_ioctl()
1120 				  job->base.bo_count,  in v3d_submit_tfu_ioctl()
1121 				  args->bo_handles[job->base.bo_count]);  in v3d_submit_tfu_ioctl()
1125 		job->base.bo[job->base.bo_count] = bo;  in v3d_submit_tfu_ioctl()
1128 	ret = v3d_lock_bo_reservations(&job->base, &acquire_ctx);  in v3d_submit_tfu_ioctl()
1133 	v3d_push_job(&job->base);  in v3d_submit_tfu_ioctl()
1137 						 &job->base, &acquire_ctx,  in v3d_submit_tfu_ioctl()
1140 						 job->base.done_fence);  in v3d_submit_tfu_ioctl()
1142 	v3d_job_put(&job->base);  in v3d_submit_tfu_ioctl()
1147 	v3d_job_cleanup((void *)job);  in v3d_submit_tfu_ioctl()
1154  * v3d_submit_csd_ioctl() - Submits a CSD (compute shader) job to the V3D.
1170 	struct v3d_csd_job *job = NULL;  in v3d_submit_csd_ioctl()  local
1199 					 &job, &clean_job, &se,  in v3d_submit_csd_ioctl()
1210 		job->base.perfmon = v3d_perfmon_find(v3d_priv,  in v3d_submit_csd_ioctl()
1212 		if (!job->base.perfmon) {  in v3d_submit_csd_ioctl()
1219 	v3d_push_job(&job->base);  in v3d_submit_csd_ioctl()
1222 					   dma_fence_get(job->base.done_fence));  in v3d_submit_csd_ioctl()
1236 	v3d_job_put(&job->base);  in v3d_submit_csd_ioctl()
1247 	v3d_job_cleanup((void *)job);  in v3d_submit_csd_ioctl()
1264  * v3d_submit_cpu_ioctl() - Submits a CPU job to the V3D.
1269  * Userspace specifies the CPU job type and data required to perform its
1303 	/* Every CPU job must have a CPU job user extension */  in v3d_submit_cpu_ioctl()
1305 		DRM_DEBUG("CPU job must have a CPU job user extension.\n");  in v3d_submit_cpu_ioctl()
1311 		DRM_DEBUG("This CPU job was not submitted with the proper number of BOs.\n");  in v3d_submit_cpu_ioctl()
1326 	csd_job = cpu_job->indirect_csd.job;  in v3d_submit_cpu_ioctl()