Lines Matching +full:width +full:- +full:mm
1 // SPDX-License-Identifier: GPL-2.0
37 if (job->out_fence) in aie2_job_release()
38 dma_fence_put(job->out_fence); in aie2_job_release()
44 kref_put(&job->refcnt, aie2_job_release); in aie2_job_put()
51 drm_sched_stop(&hwctx->priv->sched, bad_job); in aie2_hwctx_stop()
52 aie2_destroy_context(xdna->dev_handle, hwctx); in aie2_hwctx_stop()
57 struct amdxdna_gem_obj *heap = hwctx->priv->heap; in aie2_hwctx_restart()
60 ret = aie2_create_context(xdna->dev_handle, hwctx); in aie2_hwctx_restart()
66 ret = aie2_map_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, in aie2_hwctx_restart()
67 heap->mem.userptr, heap->mem.size); in aie2_hwctx_restart()
73 if (hwctx->status != HWCTX_STAT_READY) { in aie2_hwctx_restart()
74 XDNA_DBG(xdna, "hwctx is not ready, status %d", hwctx->status); in aie2_hwctx_restart()
85 drm_sched_start(&hwctx->priv->sched, 0); in aie2_hwctx_restart()
86 XDNA_DBG(xdna, "%s restarted, ret %d", hwctx->name, ret); in aie2_hwctx_restart()
92 struct amdxdna_dev *xdna = client->xdna; in aie2_restart_ctx()
96 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); in aie2_restart_ctx()
97 mutex_lock(&client->hwctx_lock); in aie2_restart_ctx()
99 if (hwctx->status != HWCTX_STAT_STOP) in aie2_restart_ctx()
102 hwctx->status = hwctx->old_status; in aie2_restart_ctx()
103 XDNA_DBG(xdna, "Resetting %s", hwctx->name); in aie2_restart_ctx()
106 mutex_unlock(&client->hwctx_lock); in aie2_restart_ctx()
114 fence = drm_syncobj_fence_get(hwctx->priv->syncobj); in aie2_cmd_get_out_fence()
133 fence = aie2_cmd_get_out_fence(hwctx, hwctx->priv->seq - 1); in aie2_hwctx_wait_for_idle()
143 struct amdxdna_dev *xdna = hwctx->client->xdna; in aie2_hwctx_suspend()
150 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); in aie2_hwctx_suspend()
153 hwctx->old_status = hwctx->status; in aie2_hwctx_suspend()
154 hwctx->status = HWCTX_STAT_STOP; in aie2_hwctx_suspend()
159 struct amdxdna_dev *xdna = hwctx->client->xdna; in aie2_hwctx_resume()
166 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); in aie2_hwctx_resume()
167 hwctx->status = hwctx->old_status; in aie2_hwctx_resume()
174 struct dma_fence *fence = job->fence; in aie2_sched_notify()
176 trace_xdna_job(&job->base, job->hwctx->name, "signaled fence", job->seq); in aie2_sched_notify()
177 job->hwctx->priv->completed++; in aie2_sched_notify()
180 up(&job->hwctx->priv->job_sem); in aie2_sched_notify()
181 job->job_done = true; in aie2_sched_notify()
183 mmput_async(job->mm); in aie2_sched_notify()
195 cmd_abo = job->cmd_bo; in aie2_sched_resp_handler()
202 ret = -EINVAL; in aie2_sched_resp_handler()
207 XDNA_DBG(job->hwctx->client->xdna, "Resp status 0x%x", status); in aie2_sched_resp_handler()
229 ret = -EINVAL; in aie2_sched_nocmd_resp_handler()
234 XDNA_DBG(job->hwctx->client->xdna, "Resp status 0x%x", status); in aie2_sched_nocmd_resp_handler()
252 cmd_abo = job->cmd_bo; in aie2_sched_cmdlist_resp_handler()
255 ret = -EINVAL; in aie2_sched_cmdlist_resp_handler()
260 xdna = job->hwctx->client->xdna; in aie2_sched_cmdlist_resp_handler()
261 XDNA_DBG(xdna, "Status 0x%x", resp->status); in aie2_sched_cmdlist_resp_handler()
262 if (resp->status == AIE2_STATUS_SUCCESS) { in aie2_sched_cmdlist_resp_handler()
268 fail_cmd_idx = resp->fail_cmd_idx; in aie2_sched_cmdlist_resp_handler()
269 fail_cmd_status = resp->fail_cmd_status; in aie2_sched_cmdlist_resp_handler()
275 ret = -EINVAL; in aie2_sched_cmdlist_resp_handler()
283 cc->error_index = fail_cmd_idx; in aie2_sched_cmdlist_resp_handler()
284 if (cc->error_index >= cc->command_count) in aie2_sched_cmdlist_resp_handler()
285 cc->error_index = 0; in aie2_sched_cmdlist_resp_handler()
296 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; in aie2_sched_job_run()
297 struct amdxdna_hwctx *hwctx = job->hwctx; in aie2_sched_job_run()
301 if (!mmget_not_zero(job->mm)) in aie2_sched_job_run()
302 return ERR_PTR(-ESRCH); in aie2_sched_job_run()
304 kref_get(&job->refcnt); in aie2_sched_job_run()
305 fence = dma_fence_get(job->fence); in aie2_sched_job_run()
323 dma_fence_put(job->fence); in aie2_sched_job_run()
325 mmput(job->mm); in aie2_sched_job_run()
328 trace_xdna_job(sched_job, hwctx->name, "sent to device", job->seq); in aie2_sched_job_run()
336 struct amdxdna_hwctx *hwctx = job->hwctx; in aie2_sched_job_free()
338 trace_xdna_job(sched_job, hwctx->name, "job free", job->seq); in aie2_sched_job_free()
339 if (!job->job_done) in aie2_sched_job_free()
340 up(&hwctx->priv->job_sem); in aie2_sched_job_free()
350 struct amdxdna_hwctx *hwctx = job->hwctx; in aie2_sched_job_timedout()
353 xdna = hwctx->client->xdna; in aie2_sched_job_timedout()
354 trace_xdna_job(sched_job, hwctx->name, "job timedout", job->seq); in aie2_sched_job_timedout()
355 mutex_lock(&xdna->dev_lock); in aie2_sched_job_timedout()
359 mutex_unlock(&xdna->dev_lock); in aie2_sched_job_timedout()
372 struct amdxdna_dev *xdna = hwctx->client->xdna; in aie2_hwctx_col_list()
375 u32 width = 1, entries = 0; in aie2_hwctx_col_list() local
378 if (!hwctx->num_tiles) { in aie2_hwctx_col_list()
380 return -EINVAL; in aie2_hwctx_col_list()
383 ndev = xdna->dev_handle; in aie2_hwctx_col_list()
384 if (unlikely(!ndev->metadata.core.row_count)) { in aie2_hwctx_col_list()
386 return -EINVAL; in aie2_hwctx_col_list()
389 hwctx->num_col = hwctx->num_tiles / ndev->metadata.core.row_count; in aie2_hwctx_col_list()
390 if (!hwctx->num_col || hwctx->num_col > ndev->total_col) { in aie2_hwctx_col_list()
391 XDNA_ERR(xdna, "Invalid num_col %d", hwctx->num_col); in aie2_hwctx_col_list()
392 return -EINVAL; in aie2_hwctx_col_list()
395 if (ndev->priv->col_align == COL_ALIGN_NATURE) in aie2_hwctx_col_list()
396 width = hwctx->num_col; in aie2_hwctx_col_list()
399 * In range [start, end], find out columns that is multiple of width. in aie2_hwctx_col_list()
404 start = xdna->dev_info->first_col; in aie2_hwctx_col_list()
405 end = ndev->total_col - hwctx->num_col; in aie2_hwctx_col_list()
410 first = start + (width - start % width) % width; in aie2_hwctx_col_list()
411 last = end - end % width; in aie2_hwctx_col_list()
413 entries = (last - first) / width + 1; in aie2_hwctx_col_list()
418 XDNA_ERR(xdna, "Start %d end %d width %d", in aie2_hwctx_col_list()
419 start, end, width); in aie2_hwctx_col_list()
420 return -EINVAL; in aie2_hwctx_col_list()
423 hwctx->col_list = kmalloc_array(entries, sizeof(*hwctx->col_list), GFP_KERNEL); in aie2_hwctx_col_list()
424 if (!hwctx->col_list) in aie2_hwctx_col_list()
425 return -ENOMEM; in aie2_hwctx_col_list()
427 hwctx->col_list_len = entries; in aie2_hwctx_col_list()
428 hwctx->col_list[0] = first; in aie2_hwctx_col_list()
430 hwctx->col_list[i] = hwctx->col_list[i - 1] + width; in aie2_hwctx_col_list()
432 print_hex_dump_debug("col_list: ", DUMP_PREFIX_OFFSET, 16, 4, hwctx->col_list, in aie2_hwctx_col_list()
433 entries * sizeof(*hwctx->col_list), false); in aie2_hwctx_col_list()
439 struct amdxdna_dev *xdna = hwctx->client->xdna; in aie2_alloc_resource()
445 return -ENOMEM; in aie2_alloc_resource()
447 xrs_req->cdo.start_cols = hwctx->col_list; in aie2_alloc_resource()
448 xrs_req->cdo.cols_len = hwctx->col_list_len; in aie2_alloc_resource()
449 xrs_req->cdo.ncols = hwctx->num_col; in aie2_alloc_resource()
450 xrs_req->cdo.qos_cap.opc = hwctx->max_opc; in aie2_alloc_resource()
452 xrs_req->rqos.gops = hwctx->qos.gops; in aie2_alloc_resource()
453 xrs_req->rqos.fps = hwctx->qos.fps; in aie2_alloc_resource()
454 xrs_req->rqos.dma_bw = hwctx->qos.dma_bandwidth; in aie2_alloc_resource()
455 xrs_req->rqos.latency = hwctx->qos.latency; in aie2_alloc_resource()
456 xrs_req->rqos.exec_time = hwctx->qos.frame_exec_time; in aie2_alloc_resource()
457 xrs_req->rqos.priority = hwctx->qos.priority; in aie2_alloc_resource()
459 xrs_req->rid = (uintptr_t)hwctx; in aie2_alloc_resource()
461 ret = xrs_allocate_resource(xdna->xrs_hdl, xrs_req, hwctx); in aie2_alloc_resource()
471 struct amdxdna_dev *xdna = hwctx->client->xdna; in aie2_release_resource()
474 ret = xrs_release_resource(xdna->xrs_hdl, (uintptr_t)hwctx); in aie2_release_resource()
481 struct amdxdna_dev *xdna = hwctx->client->xdna; in aie2_ctx_syncobj_create()
482 struct drm_file *filp = hwctx->client->filp; in aie2_ctx_syncobj_create()
487 hwctx->syncobj_hdl = AMDXDNA_INVALID_FENCE_HANDLE; in aie2_ctx_syncobj_create()
500 hwctx->priv->syncobj = syncobj; in aie2_ctx_syncobj_create()
501 hwctx->syncobj_hdl = hdl; in aie2_ctx_syncobj_create()
512 drm_syncobj_put(hwctx->priv->syncobj); in aie2_ctx_syncobj_destroy()
517 struct amdxdna_client *client = hwctx->client; in aie2_hwctx_init()
518 struct amdxdna_dev *xdna = client->xdna; in aie2_hwctx_init()
525 priv = kzalloc(sizeof(*hwctx->priv), GFP_KERNEL); in aie2_hwctx_init()
527 return -ENOMEM; in aie2_hwctx_init()
528 hwctx->priv = priv; in aie2_hwctx_init()
530 mutex_lock(&client->mm_lock); in aie2_hwctx_init()
531 heap = client->dev_heap; in aie2_hwctx_init()
534 mutex_unlock(&client->mm_lock); in aie2_hwctx_init()
535 ret = -ENOENT; in aie2_hwctx_init()
539 mutex_unlock(&client->mm_lock); in aie2_hwctx_init()
540 priv->heap = heap; in aie2_hwctx_init()
541 sema_init(&priv->job_sem, HWCTX_MAX_CMDS); in aie2_hwctx_init()
549 for (i = 0; i < ARRAY_SIZE(priv->cmd_buf); i++) { in aie2_hwctx_init()
558 abo = amdxdna_drm_alloc_dev_bo(&xdna->ddev, &args, client->filp, true); in aie2_hwctx_init()
565 i, abo->mem.dev_addr, abo->mem.size); in aie2_hwctx_init()
566 priv->cmd_buf[i] = abo; in aie2_hwctx_init()
569 sched = &priv->sched; in aie2_hwctx_init()
570 mutex_init(&priv->io_lock); in aie2_hwctx_init()
573 might_lock(&priv->io_lock); in aie2_hwctx_init()
578 NULL, NULL, hwctx->name, xdna->ddev.dev); in aie2_hwctx_init()
584 ret = drm_sched_entity_init(&priv->entity, DRM_SCHED_PRIORITY_NORMAL, in aie2_hwctx_init()
603 ret = aie2_map_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, in aie2_hwctx_init()
604 heap->mem.userptr, heap->mem.size); in aie2_hwctx_init()
616 hwctx->status = HWCTX_STAT_INIT; in aie2_hwctx_init()
617 ndev = xdna->dev_handle; in aie2_hwctx_init()
618 ndev->hwctx_num++; in aie2_hwctx_init()
620 XDNA_DBG(xdna, "hwctx %s init completed", hwctx->name); in aie2_hwctx_init()
627 kfree(hwctx->col_list); in aie2_hwctx_init()
629 drm_sched_entity_destroy(&priv->entity); in aie2_hwctx_init()
631 drm_sched_fini(&priv->sched); in aie2_hwctx_init()
633 for (i = 0; i < ARRAY_SIZE(priv->cmd_buf); i++) { in aie2_hwctx_init()
634 if (!priv->cmd_buf[i]) in aie2_hwctx_init()
636 drm_gem_object_put(to_gobj(priv->cmd_buf[i])); in aie2_hwctx_init()
652 xdna = hwctx->client->xdna; in aie2_hwctx_fini()
653 ndev = xdna->dev_handle; in aie2_hwctx_fini()
654 ndev->hwctx_num--; in aie2_hwctx_fini()
655 drm_sched_wqueue_stop(&hwctx->priv->sched); in aie2_hwctx_fini()
665 drm_sched_wqueue_start(&hwctx->priv->sched); in aie2_hwctx_fini()
668 drm_sched_entity_destroy(&hwctx->priv->entity); in aie2_hwctx_fini()
669 drm_sched_fini(&hwctx->priv->sched); in aie2_hwctx_fini()
672 XDNA_DBG(xdna, "%s sequence number %lld", hwctx->name, hwctx->priv->seq); in aie2_hwctx_fini()
674 for (idx = 0; idx < ARRAY_SIZE(hwctx->priv->cmd_buf); idx++) in aie2_hwctx_fini()
675 drm_gem_object_put(to_gobj(hwctx->priv->cmd_buf[idx])); in aie2_hwctx_fini()
676 amdxdna_gem_unpin(hwctx->priv->heap); in aie2_hwctx_fini()
677 drm_gem_object_put(to_gobj(hwctx->priv->heap)); in aie2_hwctx_fini()
679 mutex_destroy(&hwctx->priv->io_lock); in aie2_hwctx_fini()
680 kfree(hwctx->col_list); in aie2_hwctx_fini()
681 kfree(hwctx->priv); in aie2_hwctx_fini()
682 kfree(hwctx->cus); in aie2_hwctx_fini()
688 struct amdxdna_dev *xdna = hwctx->client->xdna; in aie2_hwctx_cu_config()
692 XDNA_DBG(xdna, "Config %d CU to %s", config->num_cus, hwctx->name); in aie2_hwctx_cu_config()
693 if (XDNA_MBZ_DBG(xdna, config->pad, sizeof(config->pad))) in aie2_hwctx_cu_config()
694 return -EINVAL; in aie2_hwctx_cu_config()
696 if (hwctx->status != HWCTX_STAT_INIT) { in aie2_hwctx_cu_config()
697 XDNA_ERR(xdna, "Not support re-config CU"); in aie2_hwctx_cu_config()
698 return -EINVAL; in aie2_hwctx_cu_config()
701 if (!config->num_cus) { in aie2_hwctx_cu_config()
703 return -EINVAL; in aie2_hwctx_cu_config()
706 total_size = struct_size(config, cu_configs, config->num_cus); in aie2_hwctx_cu_config()
709 return -EINVAL; in aie2_hwctx_cu_config()
712 hwctx->cus = kmemdup(config, total_size, GFP_KERNEL); in aie2_hwctx_cu_config()
713 if (!hwctx->cus) in aie2_hwctx_cu_config()
714 return -ENOMEM; in aie2_hwctx_cu_config()
723 hwctx->status = HWCTX_STAT_READY; in aie2_hwctx_cu_config()
728 kfree(hwctx->cus); in aie2_hwctx_cu_config()
729 hwctx->cus = NULL; in aie2_hwctx_cu_config()
735 struct amdxdna_dev *xdna = hwctx->client->xdna; in aie2_hwctx_config()
737 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); in aie2_hwctx_config()
743 return -EOPNOTSUPP; in aie2_hwctx_config()
746 return -EOPNOTSUPP; in aie2_hwctx_config()
752 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); in aie2_populate_range()
753 struct mm_struct *mm = abo->mem.notifier.mm; in aie2_populate_range() local
759 abo->mem.userptr, abo->mem.size); in aie2_populate_range()
760 range.notifier = &abo->mem.notifier; in aie2_populate_range()
761 range.start = abo->mem.userptr; in aie2_populate_range()
762 range.end = abo->mem.userptr + abo->mem.size; in aie2_populate_range()
763 range.hmm_pfns = abo->mem.pfns; in aie2_populate_range()
766 if (!mmget_not_zero(mm)) in aie2_populate_range()
767 return -EFAULT; in aie2_populate_range()
771 range.notifier_seq = mmu_interval_read_begin(&abo->mem.notifier); in aie2_populate_range()
772 mmap_read_lock(mm); in aie2_populate_range()
774 mmap_read_unlock(mm); in aie2_populate_range()
777 ret = -ETIME; in aie2_populate_range()
781 if (ret == -EBUSY) in aie2_populate_range()
787 down_read(&xdna->notifier_lock); in aie2_populate_range()
788 if (mmu_interval_read_retry(&abo->mem.notifier, range.notifier_seq)) { in aie2_populate_range()
789 up_read(&xdna->notifier_lock); in aie2_populate_range()
792 abo->mem.map_invalid = false; in aie2_populate_range()
793 up_read(&xdna->notifier_lock); in aie2_populate_range()
796 mmput(mm); in aie2_populate_range()
802 struct amdxdna_dev *xdna = hwctx->client->xdna; in aie2_cmd_submit()
809 ret = down_interruptible(&hwctx->priv->job_sem); in aie2_cmd_submit()
818 ret = -ENOMEM; in aie2_cmd_submit()
822 ret = drm_sched_job_init(&job->base, &hwctx->priv->entity, 1, hwctx); in aie2_cmd_submit()
829 ret = drm_gem_lock_reservations(job->bos, job->bo_cnt, &acquire_ctx); in aie2_cmd_submit()
835 for (i = 0; i < job->bo_cnt; i++) { in aie2_cmd_submit()
836 ret = dma_resv_reserve_fences(job->bos[i]->resv, 1); in aie2_cmd_submit()
839 drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); in aie2_cmd_submit()
844 down_read(&xdna->notifier_lock); in aie2_cmd_submit()
845 for (i = 0; i < job->bo_cnt; i++) { in aie2_cmd_submit()
846 abo = to_xdna_obj(job->bos[i]); in aie2_cmd_submit()
847 if (abo->mem.map_invalid) { in aie2_cmd_submit()
848 up_read(&xdna->notifier_lock); in aie2_cmd_submit()
849 drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); in aie2_cmd_submit()
854 ret = -ETIME; in aie2_cmd_submit()
865 mutex_lock(&hwctx->priv->io_lock); in aie2_cmd_submit()
866 drm_sched_job_arm(&job->base); in aie2_cmd_submit()
867 job->out_fence = dma_fence_get(&job->base.s_fence->finished); in aie2_cmd_submit()
868 for (i = 0; i < job->bo_cnt; i++) in aie2_cmd_submit()
869 dma_resv_add_fence(job->bos[i]->resv, job->out_fence, DMA_RESV_USAGE_WRITE); in aie2_cmd_submit()
870 job->seq = hwctx->priv->seq++; in aie2_cmd_submit()
871 kref_get(&job->refcnt); in aie2_cmd_submit()
872 drm_sched_entity_push_job(&job->base); in aie2_cmd_submit()
874 *seq = job->seq; in aie2_cmd_submit()
875 drm_syncobj_add_point(hwctx->priv->syncobj, chain, job->out_fence, *seq); in aie2_cmd_submit()
876 mutex_unlock(&hwctx->priv->io_lock); in aie2_cmd_submit()
878 up_read(&xdna->notifier_lock); in aie2_cmd_submit()
879 drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); in aie2_cmd_submit()
886 drm_sched_job_cleanup(&job->base); in aie2_cmd_submit()
890 up(&hwctx->priv->job_sem); in aie2_cmd_submit()
891 job->job_done = true; in aie2_cmd_submit()
898 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); in aie2_hmm_invalidate()
902 down_write(&xdna->notifier_lock); in aie2_hmm_invalidate()
903 abo->mem.map_invalid = true; in aie2_hmm_invalidate()
904 mmu_interval_set_seq(&abo->mem.notifier, cur_seq); in aie2_hmm_invalidate()
905 up_write(&xdna->notifier_lock); in aie2_hmm_invalidate()
906 ret = dma_resv_wait_timeout(gobj->resv, DMA_RESV_USAGE_BOOKKEEP, in aie2_hmm_invalidate()
908 if (!ret || ret == -ERESTARTSYS) in aie2_hmm_invalidate()