73 	struct ivpu_device *vdev = file_priv->vdev;  in ivpu_file_priv_get()  local
77 	ivpu_dbg(vdev, KREF, "file_priv get: ctx %u refcount %u\n",  in ivpu_file_priv_get()
83 static void file_priv_unbind(struct ivpu_device *vdev, struct ivpu_file_priv *file_priv)  in file_priv_unbind()  argument
87 		ivpu_dbg(vdev, FILE, "file_priv unbind: ctx %u\n", file_priv->ctx.id);  in file_priv_unbind()
90 		ivpu_bo_unbind_all_bos_from_context(vdev, &file_priv->ctx);  in file_priv_unbind()
91 		ivpu_mmu_context_fini(vdev, &file_priv->ctx);  in file_priv_unbind()
93 		drm_WARN_ON(&vdev->drm, !xa_erase_irq(&vdev->context_xa, file_priv->ctx.id));  in file_priv_unbind()
101 	struct ivpu_device *vdev = file_priv->vdev;  in file_priv_release()  local
103 	ivpu_dbg(vdev, FILE, "file_priv release: ctx %u bound %d\n",  in file_priv_release()
106 	pm_runtime_get_sync(vdev->drm.dev);  in file_priv_release()
107 	mutex_lock(&vdev->context_list_lock);  in file_priv_release()
108 	file_priv_unbind(vdev, file_priv);  in file_priv_release()
109 	drm_WARN_ON(&vdev->drm, !xa_empty(&file_priv->cmdq_xa));  in file_priv_release()
111 	mutex_unlock(&vdev->context_list_lock);  in file_priv_release()
112 	pm_runtime_put_autosuspend(vdev->drm.dev);  in file_priv_release()
122 	struct ivpu_device *vdev = file_priv->vdev;  in ivpu_file_priv_put()  local
124 	ivpu_dbg(vdev, KREF, "file_priv put: ctx %u refcount %u\n",  in ivpu_file_priv_put()
131 static int ivpu_get_capabilities(struct ivpu_device *vdev, struct drm_ivpu_param *args)  in ivpu_get_capabilities()  argument
150 	struct ivpu_device *vdev = file_priv->vdev;  in ivpu_get_param_ioctl()  local
151 	struct pci_dev *pdev = to_pci_dev(vdev->drm.dev);  in ivpu_get_param_ioctl()
167 		args->value = vdev->platform;  in ivpu_get_param_ioctl()
170 		args->value = ivpu_hw_dpu_max_freq_get(vdev);  in ivpu_get_param_ioctl()
173 		args->value = ivpu_get_context_count(vdev);  in ivpu_get_param_ioctl()
176 		args->value = vdev->hw->ranges.user.start;  in ivpu_get_param_ioctl()
185 			fw_hdr = (struct vpu_firmware_header *)vdev->fw->file->data;  in ivpu_get_param_ioctl()
192 		ret = ivpu_jsm_get_heartbeat(vdev, args->index, &args->value);  in ivpu_get_param_ioctl()
195 		args->value = (u64)atomic64_inc_return(&vdev->unique_id_counter);  in ivpu_get_param_ioctl()
198 		args->value = vdev->hw->tile_fuse;  in ivpu_get_param_ioctl()
201 		args->value = vdev->hw->sku;  in ivpu_get_param_ioctl()
204 		ret = ivpu_get_capabilities(vdev, args);  in ivpu_get_param_ioctl()
230 	struct ivpu_device *vdev = to_ivpu_device(dev);  in ivpu_open()  local
246 	file_priv->vdev = vdev;  in ivpu_open()
252 	mutex_lock(&vdev->context_list_lock);  in ivpu_open()
254 	ret = xa_alloc_irq(&vdev->context_xa, &ctx_id, file_priv,  in ivpu_open()
255 			   vdev->context_xa_limit, GFP_KERNEL);  in ivpu_open()
257 		ivpu_err(vdev, "Failed to allocate context id: %d\n", ret);  in ivpu_open()
261 	ivpu_mmu_context_init(vdev, &file_priv->ctx, ctx_id);  in ivpu_open()
270 	mutex_unlock(&vdev->context_list_lock);  in ivpu_open()
275 	ivpu_dbg(vdev, FILE, "file_priv create: ctx %u process %s pid %d\n",  in ivpu_open()
281 	mutex_unlock(&vdev->context_list_lock);  in ivpu_open()
293 	struct ivpu_device *vdev = to_ivpu_device(dev);  in ivpu_postclose()  local
295 	ivpu_dbg(vdev, FILE, "file_priv close: ctx %u process %s pid %d\n",  in ivpu_postclose()
315 static int ivpu_wait_for_ready(struct ivpu_device *vdev)  in ivpu_wait_for_ready()  argument
325 	ivpu_ipc_consumer_add(vdev, &cons, IVPU_IPC_CHAN_BOOT_MSG, NULL);  in ivpu_wait_for_ready()
327 	timeout = jiffies + msecs_to_jiffies(vdev->timeout.boot);  in ivpu_wait_for_ready()
329 		ivpu_ipc_irq_handler(vdev);  in ivpu_wait_for_ready()
330 		ret = ivpu_ipc_receive(vdev, &cons, &ipc_hdr, NULL, 0);  in ivpu_wait_for_ready()
337 	ivpu_ipc_consumer_del(vdev, &cons);  in ivpu_wait_for_ready()
340 		ivpu_err(vdev, "Invalid NPU ready message: 0x%x\n",  in ivpu_wait_for_ready()
346 		ivpu_dbg(vdev, PM, "NPU ready message received successfully\n");  in ivpu_wait_for_ready()
351 static int ivpu_hw_sched_init(struct ivpu_device *vdev)  in ivpu_hw_sched_init()  argument
355 	if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW) {  in ivpu_hw_sched_init()
356 		ret = ivpu_jsm_hws_setup_priority_bands(vdev);  in ivpu_hw_sched_init()
358 			ivpu_err(vdev, "Failed to enable hw scheduler: %d", ret);  in ivpu_hw_sched_init()
368  * @vdev: VPU device
373 int ivpu_boot(struct ivpu_device *vdev)  in ivpu_boot()  argument
378 	ivpu_fw_boot_params_setup(vdev, ivpu_bo_vaddr(vdev->fw->mem));  in ivpu_boot()
380 	ret = ivpu_hw_boot_fw(vdev);  in ivpu_boot()
382 		ivpu_err(vdev, "Failed to start the firmware: %d\n", ret);  in ivpu_boot()
386 	ret = ivpu_wait_for_ready(vdev);  in ivpu_boot()
388 		ivpu_err(vdev, "Failed to boot the firmware: %d\n", ret);  in ivpu_boot()
392 	ivpu_hw_irq_clear(vdev);  in ivpu_boot()
393 	enable_irq(vdev->irq);  in ivpu_boot()
394 	ivpu_hw_irq_enable(vdev);  in ivpu_boot()
395 	ivpu_ipc_enable(vdev);  in ivpu_boot()
397 	if (ivpu_fw_is_cold_boot(vdev)) {  in ivpu_boot()
398 		ret = ivpu_pm_dct_init(vdev);  in ivpu_boot()
402 		ret = ivpu_hw_sched_init(vdev);  in ivpu_boot()
410 	ivpu_ipc_disable(vdev);  in ivpu_boot()
411 	ivpu_hw_irq_disable(vdev);  in ivpu_boot()
412 	disable_irq(vdev->irq);  in ivpu_boot()
414 	ivpu_hw_diagnose_failure(vdev);  in ivpu_boot()
415 	ivpu_mmu_evtq_dump(vdev);  in ivpu_boot()
416 	ivpu_dev_coredump(vdev);  in ivpu_boot()
420 void ivpu_prepare_for_reset(struct ivpu_device *vdev)  in ivpu_prepare_for_reset()  argument
422 	ivpu_hw_irq_disable(vdev);  in ivpu_prepare_for_reset()
423 	disable_irq(vdev->irq);  in ivpu_prepare_for_reset()
424 	ivpu_ipc_disable(vdev);  in ivpu_prepare_for_reset()
425 	ivpu_mmu_disable(vdev);  in ivpu_prepare_for_reset()
428 int ivpu_shutdown(struct ivpu_device *vdev)  in ivpu_shutdown()  argument
433 	pci_save_state(to_pci_dev(vdev->drm.dev));  in ivpu_shutdown()
435 	ret = ivpu_hw_power_down(vdev);  in ivpu_shutdown()
437 		ivpu_warn(vdev, "Failed to power down HW: %d\n", ret);  in ivpu_shutdown()
439 	pci_set_power_state(to_pci_dev(vdev->drm.dev), PCI_D3hot);  in ivpu_shutdown()
468 static void ivpu_context_abort_invalid(struct ivpu_device *vdev)  in ivpu_context_abort_invalid()  argument
473 	mutex_lock(&vdev->context_list_lock);  in ivpu_context_abort_invalid()
475 	xa_for_each(&vdev->context_xa, ctx_id, file_priv) {  in ivpu_context_abort_invalid()
485 	mutex_unlock(&vdev->context_list_lock);  in ivpu_context_abort_invalid()
490 	struct ivpu_device *vdev = arg;  in ivpu_irq_thread_handler()  local
493 	if (kfifo_is_empty(&vdev->hw->irq.fifo))  in ivpu_irq_thread_handler()
496 	while (kfifo_get(&vdev->hw->irq.fifo, &irq_src)) {  in ivpu_irq_thread_handler()
499 			ivpu_ipc_irq_thread_handler(vdev);  in ivpu_irq_thread_handler()
502 			ivpu_context_abort_invalid(vdev);  in ivpu_irq_thread_handler()
505 			ivpu_pm_dct_irq_thread_handler(vdev);  in ivpu_irq_thread_handler()
508 			ivpu_err_ratelimited(vdev, "Unknown IRQ source: %u\n", irq_src);  in ivpu_irq_thread_handler()
516 static int ivpu_irq_init(struct ivpu_device *vdev)  in ivpu_irq_init()  argument
518 	struct pci_dev *pdev = to_pci_dev(vdev->drm.dev);  in ivpu_irq_init()
523 		ivpu_err(vdev, "Failed to allocate a MSI IRQ: %d\n", ret);  in ivpu_irq_init()
527 	ivpu_irq_handlers_init(vdev);  in ivpu_irq_init()
529 	vdev->irq = pci_irq_vector(pdev, 0);  in ivpu_irq_init()
531 	ret = devm_request_threaded_irq(vdev->drm.dev, vdev->irq, ivpu_hw_irq_handler,  in ivpu_irq_init()
532 					ivpu_irq_thread_handler, IRQF_NO_AUTOEN, DRIVER_NAME, vdev);  in ivpu_irq_init()
534 		ivpu_err(vdev, "Failed to request an IRQ %d\n", ret);  in ivpu_irq_init()
539 static int ivpu_pci_init(struct ivpu_device *vdev)  in ivpu_pci_init()  argument
541 	struct pci_dev *pdev = to_pci_dev(vdev->drm.dev);  in ivpu_pci_init()
546 	ivpu_dbg(vdev, MISC, "Mapping BAR0 (RegV) %pR\n", bar0);  in ivpu_pci_init()
547 	vdev->regv = devm_ioremap_resource(vdev->drm.dev, bar0);  in ivpu_pci_init()
548 	if (IS_ERR(vdev->regv)) {  in ivpu_pci_init()
549 		ivpu_err(vdev, "Failed to map bar 0: %pe\n", vdev->regv);  in ivpu_pci_init()
550 		return PTR_ERR(vdev->regv);  in ivpu_pci_init()
553 	ivpu_dbg(vdev, MISC, "Mapping BAR4 (RegB) %pR\n", bar4);  in ivpu_pci_init()
554 	vdev->regb = devm_ioremap_resource(vdev->drm.dev, bar4);  in ivpu_pci_init()
555 	if (IS_ERR(vdev->regb)) {  in ivpu_pci_init()
556 		ivpu_err(vdev, "Failed to map bar 4: %pe\n", vdev->regb);  in ivpu_pci_init()
557 		return PTR_ERR(vdev->regb);  in ivpu_pci_init()
560 	ret = dma_set_mask_and_coherent(vdev->drm.dev, DMA_BIT_MASK(vdev->hw->dma_bits));  in ivpu_pci_init()
562 		ivpu_err(vdev, "Failed to set DMA mask: %d\n", ret);  in ivpu_pci_init()
565 	dma_set_max_seg_size(vdev->drm.dev, UINT_MAX);  in ivpu_pci_init()
575 		ivpu_err(vdev, "Failed to enable PCI device: %d\n", ret);  in ivpu_pci_init()
584 static int ivpu_dev_init(struct ivpu_device *vdev)  in ivpu_dev_init()  argument
588 	vdev->hw = drmm_kzalloc(&vdev->drm, sizeof(*vdev->hw), GFP_KERNEL);  in ivpu_dev_init()
589 	if (!vdev->hw)  in ivpu_dev_init()
592 	vdev->mmu = drmm_kzalloc(&vdev->drm, sizeof(*vdev->mmu), GFP_KERNEL);  in ivpu_dev_init()
593 	if (!vdev->mmu)  in ivpu_dev_init()
596 	vdev->fw = drmm_kzalloc(&vdev->drm, sizeof(*vdev->fw), GFP_KERNEL);  in ivpu_dev_init()
597 	if (!vdev->fw)  in ivpu_dev_init()
600 	vdev->ipc = drmm_kzalloc(&vdev->drm, sizeof(*vdev->ipc), GFP_KERNEL);  in ivpu_dev_init()
601 	if (!vdev->ipc)  in ivpu_dev_init()
604 	vdev->pm = drmm_kzalloc(&vdev->drm, sizeof(*vdev->pm), GFP_KERNEL);  in ivpu_dev_init()
605 	if (!vdev->pm)  in ivpu_dev_init()
608 	if (ivpu_hw_ip_gen(vdev) >= IVPU_HW_IP_40XX)  in ivpu_dev_init()
609 		vdev->hw->dma_bits = 48;  in ivpu_dev_init()
611 		vdev->hw->dma_bits = 38;  in ivpu_dev_init()
613 	vdev->platform = IVPU_PLATFORM_INVALID;  in ivpu_dev_init()
614 	vdev->context_xa_limit.min = IVPU_USER_CONTEXT_MIN_SSID;  in ivpu_dev_init()
615 	vdev->context_xa_limit.max = IVPU_USER_CONTEXT_MAX_SSID;  in ivpu_dev_init()
616 	atomic64_set(&vdev->unique_id_counter, 0);  in ivpu_dev_init()
617 	xa_init_flags(&vdev->context_xa, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);  in ivpu_dev_init()
618 	xa_init_flags(&vdev->submitted_jobs_xa, XA_FLAGS_ALLOC1);  in ivpu_dev_init()
619 	xa_init_flags(&vdev->db_xa, XA_FLAGS_ALLOC1);  in ivpu_dev_init()
620 	lockdep_set_class(&vdev->submitted_jobs_xa.xa_lock, &submitted_jobs_xa_lock_class_key);  in ivpu_dev_init()
621 	INIT_LIST_HEAD(&vdev->bo_list);  in ivpu_dev_init()
623 	vdev->db_limit.min = IVPU_MIN_DB;  in ivpu_dev_init()
624 	vdev->db_limit.max = IVPU_MAX_DB;  in ivpu_dev_init()
626 	ret = drmm_mutex_init(&vdev->drm, &vdev->context_list_lock);  in ivpu_dev_init()
630 	ret = drmm_mutex_init(&vdev->drm, &vdev->bo_list_lock);  in ivpu_dev_init()
634 	ret = ivpu_pci_init(vdev);  in ivpu_dev_init()
638 	ret = ivpu_irq_init(vdev);  in ivpu_dev_init()
643 	ret = ivpu_hw_init(vdev);  in ivpu_dev_init()
648 	ret = ivpu_hw_power_up(vdev);  in ivpu_dev_init()
652 	ivpu_mmu_global_context_init(vdev);  in ivpu_dev_init()
654 	ret = ivpu_mmu_init(vdev);  in ivpu_dev_init()
658 	ret = ivpu_mmu_reserved_context_init(vdev);  in ivpu_dev_init()
662 	ret = ivpu_fw_init(vdev);  in ivpu_dev_init()
666 	ret = ivpu_ipc_init(vdev);  in ivpu_dev_init()
670 	ivpu_pm_init(vdev);  in ivpu_dev_init()
672 	ret = ivpu_boot(vdev);  in ivpu_dev_init()
676 	ivpu_job_done_consumer_init(vdev);  in ivpu_dev_init()
677 	ivpu_pm_enable(vdev);  in ivpu_dev_init()
682 	ivpu_ipc_fini(vdev);  in ivpu_dev_init()
684 	ivpu_fw_fini(vdev);  in ivpu_dev_init()
686 	ivpu_mmu_reserved_context_fini(vdev);  in ivpu_dev_init()
688 	ivpu_mmu_global_context_fini(vdev);  in ivpu_dev_init()
690 	ivpu_shutdown(vdev);  in ivpu_dev_init()
692 	xa_destroy(&vdev->db_xa);  in ivpu_dev_init()
693 	xa_destroy(&vdev->submitted_jobs_xa);  in ivpu_dev_init()
694 	xa_destroy(&vdev->context_xa);  in ivpu_dev_init()
698 static void ivpu_bo_unbind_all_user_contexts(struct ivpu_device *vdev)  in ivpu_bo_unbind_all_user_contexts()  argument
703 	mutex_lock(&vdev->context_list_lock);  in ivpu_bo_unbind_all_user_contexts()
705 	xa_for_each(&vdev->context_xa, ctx_id, file_priv)  in ivpu_bo_unbind_all_user_contexts()
706 		file_priv_unbind(vdev, file_priv);  in ivpu_bo_unbind_all_user_contexts()
708 	mutex_unlock(&vdev->context_list_lock);  in ivpu_bo_unbind_all_user_contexts()
711 static void ivpu_dev_fini(struct ivpu_device *vdev)  in ivpu_dev_fini()  argument
713 	ivpu_jobs_abort_all(vdev);  in ivpu_dev_fini()
714 	ivpu_pm_cancel_recovery(vdev);  in ivpu_dev_fini()
715 	ivpu_pm_disable(vdev);  in ivpu_dev_fini()
716 	ivpu_prepare_for_reset(vdev);  in ivpu_dev_fini()
717 	ivpu_shutdown(vdev);  in ivpu_dev_fini()
719 	ivpu_ms_cleanup_all(vdev);  in ivpu_dev_fini()
720 	ivpu_job_done_consumer_fini(vdev);  in ivpu_dev_fini()
721 	ivpu_bo_unbind_all_user_contexts(vdev);  in ivpu_dev_fini()
723 	ivpu_ipc_fini(vdev);  in ivpu_dev_fini()
724 	ivpu_fw_fini(vdev);  in ivpu_dev_fini()
725 	ivpu_mmu_reserved_context_fini(vdev);  in ivpu_dev_fini()
726 	ivpu_mmu_global_context_fini(vdev);  in ivpu_dev_fini()
728 	drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->db_xa));  in ivpu_dev_fini()
729 	xa_destroy(&vdev->db_xa);  in ivpu_dev_fini()
730 	drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->submitted_jobs_xa));  in ivpu_dev_fini()
731 	xa_destroy(&vdev->submitted_jobs_xa);  in ivpu_dev_fini()
732 	drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->context_xa));  in ivpu_dev_fini()
733 	xa_destroy(&vdev->context_xa);  in ivpu_dev_fini()
747 	struct ivpu_device *vdev;  in ivpu_probe()  local
750 	vdev = devm_drm_dev_alloc(&pdev->dev, &driver, struct ivpu_device, drm);  in ivpu_probe()
751 	if (IS_ERR(vdev))  in ivpu_probe()
752 		return PTR_ERR(vdev);  in ivpu_probe()
754 	pci_set_drvdata(pdev, vdev);  in ivpu_probe()
756 	ret = ivpu_dev_init(vdev);  in ivpu_probe()
760 	ivpu_debugfs_init(vdev);  in ivpu_probe()
761 	ivpu_sysfs_init(vdev);  in ivpu_probe()
763 	ret = drm_dev_register(&vdev->drm, 0);  in ivpu_probe()
766 		ivpu_dev_fini(vdev);  in ivpu_probe()
774 	struct ivpu_device *vdev = pci_get_drvdata(pdev);  in ivpu_remove()  local
776 	drm_dev_unplug(&vdev->drm);  in ivpu_remove()
777 	ivpu_dev_fini(vdev);  in ivpu_remove()