38 static void ivpu_pm_prepare_cold_boot(struct ivpu_device *vdev)  in ivpu_pm_prepare_cold_boot()  argument
40 	struct ivpu_fw_info *fw = vdev->fw;  in ivpu_pm_prepare_cold_boot()
42 	ivpu_cmdq_reset_all_contexts(vdev);  in ivpu_pm_prepare_cold_boot()
43 	ivpu_ipc_reset(vdev);  in ivpu_pm_prepare_cold_boot()
44 	ivpu_fw_log_reset(vdev);  in ivpu_pm_prepare_cold_boot()
45 	ivpu_fw_load(vdev);  in ivpu_pm_prepare_cold_boot()
49 static void ivpu_pm_prepare_warm_boot(struct ivpu_device *vdev)  in ivpu_pm_prepare_warm_boot()  argument
51 	struct ivpu_fw_info *fw = vdev->fw;  in ivpu_pm_prepare_warm_boot()
55 		ivpu_pm_prepare_cold_boot(vdev);  in ivpu_pm_prepare_warm_boot()
59 	ivpu_dbg(vdev, FW_BOOT, "Save/restore entry point %llx", bp->save_restore_ret_address);  in ivpu_pm_prepare_warm_boot()
63 static int ivpu_suspend(struct ivpu_device *vdev)  in ivpu_suspend()  argument
67 	ivpu_prepare_for_reset(vdev);  in ivpu_suspend()
69 	ret = ivpu_shutdown(vdev);  in ivpu_suspend()
71 		ivpu_err(vdev, "Failed to shutdown NPU: %d\n", ret);  in ivpu_suspend()
76 static int ivpu_resume(struct ivpu_device *vdev)  in ivpu_resume()  argument
81 	pci_set_power_state(to_pci_dev(vdev->drm.dev), PCI_D0);  in ivpu_resume()
82 	pci_restore_state(to_pci_dev(vdev->drm.dev));  in ivpu_resume()
84 	ret = ivpu_hw_power_up(vdev);  in ivpu_resume()
86 		ivpu_err(vdev, "Failed to power up HW: %d\n", ret);  in ivpu_resume()
90 	ret = ivpu_mmu_enable(vdev);  in ivpu_resume()
92 		ivpu_err(vdev, "Failed to resume MMU: %d\n", ret);  in ivpu_resume()
96 	ret = ivpu_boot(vdev);  in ivpu_resume()
103 	ivpu_mmu_disable(vdev);  in ivpu_resume()
105 	ivpu_hw_power_down(vdev);  in ivpu_resume()
106 	pci_set_power_state(to_pci_dev(vdev->drm.dev), PCI_D3hot);  in ivpu_resume()
108 	if (!ivpu_fw_is_cold_boot(vdev)) {  in ivpu_resume()
109 		ivpu_pm_prepare_cold_boot(vdev);  in ivpu_resume()
112 		ivpu_err(vdev, "Failed to resume the FW: %d\n", ret);  in ivpu_resume()
118 static void ivpu_pm_reset_begin(struct ivpu_device *vdev)  in ivpu_pm_reset_begin()  argument
120 	pm_runtime_disable(vdev->drm.dev);  in ivpu_pm_reset_begin()
122 	atomic_inc(&vdev->pm->reset_counter);  in ivpu_pm_reset_begin()
123 	atomic_set(&vdev->pm->reset_pending, 1);  in ivpu_pm_reset_begin()
124 	down_write(&vdev->pm->reset_lock);  in ivpu_pm_reset_begin()
127 static void ivpu_pm_reset_complete(struct ivpu_device *vdev)  in ivpu_pm_reset_complete()  argument
131 	ivpu_pm_prepare_cold_boot(vdev);  in ivpu_pm_reset_complete()
132 	ivpu_jobs_abort_all(vdev);  in ivpu_pm_reset_complete()
133 	ivpu_ms_cleanup_all(vdev);  in ivpu_pm_reset_complete()
135 	ret = ivpu_resume(vdev);  in ivpu_pm_reset_complete()
137 		ivpu_err(vdev, "Failed to resume NPU: %d\n", ret);  in ivpu_pm_reset_complete()
138 		pm_runtime_set_suspended(vdev->drm.dev);  in ivpu_pm_reset_complete()
140 		pm_runtime_set_active(vdev->drm.dev);  in ivpu_pm_reset_complete()
143 	up_write(&vdev->pm->reset_lock);  in ivpu_pm_reset_complete()
144 	atomic_set(&vdev->pm->reset_pending, 0);  in ivpu_pm_reset_complete()
146 	pm_runtime_mark_last_busy(vdev->drm.dev);  in ivpu_pm_reset_complete()
147 	pm_runtime_enable(vdev->drm.dev);  in ivpu_pm_reset_complete()
153 	struct ivpu_device *vdev = pm->vdev;  in ivpu_pm_recovery_work()  local
156 	ivpu_err(vdev, "Recovering the NPU (reset #%d)\n", atomic_read(&vdev->pm->reset_counter));  in ivpu_pm_recovery_work()
158 	ivpu_pm_reset_begin(vdev);  in ivpu_pm_recovery_work()
160 	if (!pm_runtime_status_suspended(vdev->drm.dev)) {  in ivpu_pm_recovery_work()
161 		ivpu_jsm_state_dump(vdev);  in ivpu_pm_recovery_work()
162 		ivpu_dev_coredump(vdev);  in ivpu_pm_recovery_work()
163 		ivpu_suspend(vdev);  in ivpu_pm_recovery_work()
166 	ivpu_pm_reset_complete(vdev);  in ivpu_pm_recovery_work()
168 	kobject_uevent_env(&vdev->drm.dev->kobj, KOBJ_CHANGE, evt);  in ivpu_pm_recovery_work()
171 void ivpu_pm_trigger_recovery(struct ivpu_device *vdev, const char *reason)  in ivpu_pm_trigger_recovery()  argument
173 	ivpu_err(vdev, "Recovery triggered by %s\n", reason);  in ivpu_pm_trigger_recovery()
176 		ivpu_err(vdev, "Recovery not available when disable_recovery param is set\n");  in ivpu_pm_trigger_recovery()
180 	if (ivpu_is_fpga(vdev)) {  in ivpu_pm_trigger_recovery()
181 		ivpu_err(vdev, "Recovery not available on FPGA\n");  in ivpu_pm_trigger_recovery()
186 	if (atomic_cmpxchg(&vdev->pm->reset_pending, 0, 1) == 0) {  in ivpu_pm_trigger_recovery()
187 		ivpu_hw_diagnose_failure(vdev);  in ivpu_pm_trigger_recovery()
188 		ivpu_hw_irq_disable(vdev); /* Disable IRQ early to protect from IRQ storm */  in ivpu_pm_trigger_recovery()
189 		queue_work(system_long_wq, &vdev->pm->recovery_work);  in ivpu_pm_trigger_recovery()
196 	struct ivpu_device *vdev = pm->vdev;  in ivpu_job_timeout_work()  local
198 	ivpu_pm_trigger_recovery(vdev, "TDR");  in ivpu_job_timeout_work()
201 void ivpu_start_job_timeout_detection(struct ivpu_device *vdev)  in ivpu_start_job_timeout_detection()  argument
203 	unsigned long timeout_ms = ivpu_tdr_timeout_ms ? ivpu_tdr_timeout_ms : vdev->timeout.tdr;  in ivpu_start_job_timeout_detection()
206 	queue_delayed_work(system_wq, &vdev->pm->job_timeout_work, msecs_to_jiffies(timeout_ms));  in ivpu_start_job_timeout_detection()
209 void ivpu_stop_job_timeout_detection(struct ivpu_device *vdev)  in ivpu_stop_job_timeout_detection()  argument
211 	cancel_delayed_work_sync(&vdev->pm->job_timeout_work);  in ivpu_stop_job_timeout_detection()
217 	struct ivpu_device *vdev = to_ivpu_device(drm);  in ivpu_pm_suspend_cb()  local
221 	ivpu_dbg(vdev, PM, "Suspend..\n");  in ivpu_pm_suspend_cb()
223 	timeout = jiffies + msecs_to_jiffies(vdev->timeout.tdr);  in ivpu_pm_suspend_cb()
224 	while (!ivpu_hw_is_idle(vdev)) {  in ivpu_pm_suspend_cb()
227 			ivpu_err(vdev, "Failed to enter idle on system suspend\n");  in ivpu_pm_suspend_cb()
232 	ivpu_jsm_pwr_d0i3_enter(vdev);  in ivpu_pm_suspend_cb()
234 	ivpu_suspend(vdev);  in ivpu_pm_suspend_cb()
235 	ivpu_pm_prepare_warm_boot(vdev);  in ivpu_pm_suspend_cb()
237 	ivpu_dbg(vdev, PM, "Suspend done.\n");  in ivpu_pm_suspend_cb()
246 	struct ivpu_device *vdev = to_ivpu_device(drm);  in ivpu_pm_resume_cb()  local
250 	ivpu_dbg(vdev, PM, "Resume..\n");  in ivpu_pm_resume_cb()
252 	ret = ivpu_resume(vdev);  in ivpu_pm_resume_cb()
254 		ivpu_err(vdev, "Failed to resume: %d\n", ret);  in ivpu_pm_resume_cb()
256 	ivpu_dbg(vdev, PM, "Resume done.\n");  in ivpu_pm_resume_cb()
265 	struct ivpu_device *vdev = to_ivpu_device(drm);  in ivpu_pm_runtime_suspend_cb()  local
269 	drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->submitted_jobs_xa));  in ivpu_pm_runtime_suspend_cb()
270 	drm_WARN_ON(&vdev->drm, work_pending(&vdev->pm->recovery_work));  in ivpu_pm_runtime_suspend_cb()
273 	ivpu_dbg(vdev, PM, "Runtime suspend..\n");  in ivpu_pm_runtime_suspend_cb()
275 	ivpu_mmu_disable(vdev);  in ivpu_pm_runtime_suspend_cb()
277 	is_idle = ivpu_hw_is_idle(vdev) || vdev->pm->dct_active_percent;  in ivpu_pm_runtime_suspend_cb()
279 		ivpu_err(vdev, "NPU is not idle before autosuspend\n");  in ivpu_pm_runtime_suspend_cb()
281 	ret_d0i3 = ivpu_jsm_pwr_d0i3_enter(vdev);  in ivpu_pm_runtime_suspend_cb()
283 		ivpu_err(vdev, "Failed to prepare for d0i3: %d\n", ret_d0i3);  in ivpu_pm_runtime_suspend_cb()
285 	ret = ivpu_suspend(vdev);  in ivpu_pm_runtime_suspend_cb()
287 		ivpu_err(vdev, "Failed to suspend NPU: %d\n", ret);  in ivpu_pm_runtime_suspend_cb()
290 		ivpu_err(vdev, "Forcing cold boot due to previous errors\n");  in ivpu_pm_runtime_suspend_cb()
291 		atomic_inc(&vdev->pm->reset_counter);  in ivpu_pm_runtime_suspend_cb()
292 		ivpu_dev_coredump(vdev);  in ivpu_pm_runtime_suspend_cb()
293 		ivpu_pm_prepare_cold_boot(vdev);  in ivpu_pm_runtime_suspend_cb()
295 		ivpu_pm_prepare_warm_boot(vdev);  in ivpu_pm_runtime_suspend_cb()
298 	ivpu_dbg(vdev, PM, "Runtime suspend done.\n");  in ivpu_pm_runtime_suspend_cb()
307 	struct ivpu_device *vdev = to_ivpu_device(drm);  in ivpu_pm_runtime_resume_cb()  local
311 	ivpu_dbg(vdev, PM, "Runtime resume..\n");  in ivpu_pm_runtime_resume_cb()
313 	ret = ivpu_resume(vdev);  in ivpu_pm_runtime_resume_cb()
315 		ivpu_err(vdev, "Failed to set RESUME state: %d\n", ret);  in ivpu_pm_runtime_resume_cb()
317 	ivpu_dbg(vdev, PM, "Runtime resume done.\n");  in ivpu_pm_runtime_resume_cb()
323 int ivpu_rpm_get(struct ivpu_device *vdev)  in ivpu_rpm_get()  argument
327 	ret = pm_runtime_resume_and_get(vdev->drm.dev);  in ivpu_rpm_get()
329 		ivpu_err(vdev, "Failed to resume NPU: %d\n", ret);  in ivpu_rpm_get()
330 		pm_runtime_set_suspended(vdev->drm.dev);  in ivpu_rpm_get()
336 void ivpu_rpm_put(struct ivpu_device *vdev)  in ivpu_rpm_put()  argument
338 	pm_runtime_mark_last_busy(vdev->drm.dev);  in ivpu_rpm_put()
339 	pm_runtime_put_autosuspend(vdev->drm.dev);  in ivpu_rpm_put()
344 	struct ivpu_device *vdev = pci_get_drvdata(pdev);  in ivpu_pm_reset_prepare_cb()  local
346 	ivpu_dbg(vdev, PM, "Pre-reset..\n");  in ivpu_pm_reset_prepare_cb()
348 	ivpu_pm_reset_begin(vdev);  in ivpu_pm_reset_prepare_cb()
350 	if (!pm_runtime_status_suspended(vdev->drm.dev)) {  in ivpu_pm_reset_prepare_cb()
351 		ivpu_prepare_for_reset(vdev);  in ivpu_pm_reset_prepare_cb()
352 		ivpu_hw_reset(vdev);  in ivpu_pm_reset_prepare_cb()
355 	ivpu_dbg(vdev, PM, "Pre-reset done.\n");  in ivpu_pm_reset_prepare_cb()
360 	struct ivpu_device *vdev = pci_get_drvdata(pdev);  in ivpu_pm_reset_done_cb()  local
362 	ivpu_dbg(vdev, PM, "Post-reset..\n");  in ivpu_pm_reset_done_cb()
364 	ivpu_pm_reset_complete(vdev);  in ivpu_pm_reset_done_cb()
366 	ivpu_dbg(vdev, PM, "Post-reset done.\n");  in ivpu_pm_reset_done_cb()
369 void ivpu_pm_init(struct ivpu_device *vdev)  in ivpu_pm_init()  argument
371 	struct device *dev = vdev->drm.dev;  in ivpu_pm_init()
372 	struct ivpu_pm_info *pm = vdev->pm;  in ivpu_pm_init()
375 	pm->vdev = vdev;  in ivpu_pm_init()
387 		delay = vdev->timeout.autosuspend;  in ivpu_pm_init()
393 	ivpu_dbg(vdev, PM, "Autosuspend delay = %d\n", delay);  in ivpu_pm_init()
396 void ivpu_pm_cancel_recovery(struct ivpu_device *vdev)  in ivpu_pm_cancel_recovery()  argument
398 	drm_WARN_ON(&vdev->drm, delayed_work_pending(&vdev->pm->job_timeout_work));  in ivpu_pm_cancel_recovery()
399 	cancel_work_sync(&vdev->pm->recovery_work);  in ivpu_pm_cancel_recovery()
402 void ivpu_pm_enable(struct ivpu_device *vdev)  in ivpu_pm_enable()  argument
404 	struct device *dev = vdev->drm.dev;  in ivpu_pm_enable()
411 void ivpu_pm_disable(struct ivpu_device *vdev)  in ivpu_pm_disable()  argument
413 	pm_runtime_get_noresume(vdev->drm.dev);  in ivpu_pm_disable()
414 	pm_runtime_forbid(vdev->drm.dev);  in ivpu_pm_disable()
417 int ivpu_pm_dct_init(struct ivpu_device *vdev)  in ivpu_pm_dct_init()  argument
419 	if (vdev->pm->dct_active_percent)  in ivpu_pm_dct_init()
420 		return ivpu_pm_dct_enable(vdev, vdev->pm->dct_active_percent);  in ivpu_pm_dct_init()
425 int ivpu_pm_dct_enable(struct ivpu_device *vdev, u8 active_percent)  in ivpu_pm_dct_enable()  argument
436 	ret = ivpu_jsm_dct_enable(vdev, active_us, inactive_us);  in ivpu_pm_dct_enable()
438 		ivpu_err_ratelimited(vdev, "Failed to enable DCT: %d\n", ret);  in ivpu_pm_dct_enable()
442 	vdev->pm->dct_active_percent = active_percent;  in ivpu_pm_dct_enable()
444 	ivpu_dbg(vdev, PM, "DCT set to %u%% (D0: %uus, D0i2: %uus)\n",  in ivpu_pm_dct_enable()
449 int ivpu_pm_dct_disable(struct ivpu_device *vdev)  in ivpu_pm_dct_disable()  argument
453 	ret = ivpu_jsm_dct_disable(vdev);  in ivpu_pm_dct_disable()
455 		ivpu_err_ratelimited(vdev, "Failed to disable DCT: %d\n", ret);  in ivpu_pm_dct_disable()
459 	vdev->pm->dct_active_percent = 0;  in ivpu_pm_dct_disable()
461 	ivpu_dbg(vdev, PM, "DCT disabled\n");  in ivpu_pm_dct_disable()
465 void ivpu_pm_dct_irq_thread_handler(struct ivpu_device *vdev)  in ivpu_pm_dct_irq_thread_handler()  argument
470 	if (ivpu_hw_btrs_dct_get_request(vdev, &enable))  in ivpu_pm_dct_irq_thread_handler()
473 	if (vdev->pm->dct_active_percent)  in ivpu_pm_dct_irq_thread_handler()
474 		ret = ivpu_pm_dct_enable(vdev, DCT_DEFAULT_ACTIVE_PERCENT);  in ivpu_pm_dct_irq_thread_handler()
476 		ret = ivpu_pm_dct_disable(vdev);  in ivpu_pm_dct_irq_thread_handler()
479 		ivpu_hw_btrs_dct_set_status(vdev, enable, vdev->pm->dct_active_percent);  in ivpu_pm_dct_irq_thread_handler()