1 // SPDX-License-Identifier: GPL-2.0-only
3  * Copyright (C) 2013--2024 Intel Corporation
8 #include <linux/dma-mapping.h>
16 #include <linux/pci-ats.h>
23 #include <media/ipu-bridge.h>
24 #include <media/ipu6-pci-table.h>
27 #include "ipu6-bus.h"
28 #include "ipu6-buttress.h"
29 #include "ipu6-cpd.h"
30 #include "ipu6-isys.h"
31 #include "ipu6-mmu.h"
32 #include "ipu6-platform-buttress-regs.h"
33 #include "ipu6-platform-isys-csi2-reg.h"
34 #include "ipu6-platform-regs.h"
70 	u32 dummy[2];
83 					3, 8, 2, 2, 2, 2, 2, 2, 1, 1,
88 					2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
89 					2, 2, 2, 2, 2, 2
102 					2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
103 					2, 2, 2, 1, 1, 4
107 					2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
108 					2, 2, 2, 2, 2, 2
125 		.cdc_fifo_threshold = {6, 8, 2},
143 					2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
144 					2, 2, 2, 2, 2, 2
148 					2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
149 					2, 2, 2, 2, 2, 2
162 					1, 2, 2, 2, 2, 2, 2, 2, 2, 2,
163 					2, 2, 2, 2, 2, 10,
165 					4, 2, 1, 1, 1, 1, 14
169 					2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
170 					2, 2, 2, 2, 2, 2,
171 					2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
172 					2, 2, 2, 2, 2, 2
186 					16, 16, 2, 2, 2, 1, 12
190 					2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
191 					2, 2, 2, 2, 2, 2
243 	server_fw_addr = lower_32_bits(*(pkg_dir + (pkg_dir_idx + 1) * 2));  in ipu6_pkg_dir_configure_spc()
245 		dma_addr = sg_dma_address(isp->isys->fw_sgt.sgl);  in ipu6_pkg_dir_configure_spc()
247 		dma_addr = sg_dma_address(isp->psys->fw_sgt.sgl);  in ipu6_pkg_dir_configure_spc()
249 	pg_offset = server_fw_addr - dma_addr;  in ipu6_pkg_dir_configure_spc()
250 	prog = (struct ipu6_cell_program *)((uintptr_t)isp->cpd_fw->data +  in ipu6_pkg_dir_configure_spc()
252 	spc_base = base + prog->regs_addr;  in ipu6_pkg_dir_configure_spc()
253 	if (spc_base != (base + hw_variant->spc_offset))  in ipu6_pkg_dir_configure_spc()
254 		dev_warn(&isp->pdev->dev,  in ipu6_pkg_dir_configure_spc()
256 			 base + hw_variant->spc_offset, spc_base);  in ipu6_pkg_dir_configure_spc()
257 	writel(server_fw_addr + prog->blob_offset +  in ipu6_pkg_dir_configure_spc()
258 	       prog->icache_source, spc_base + IPU6_PSYS_REG_SPC_ICACHE_BASE);  in ipu6_pkg_dir_configure_spc()
261 	writel(prog->start[1], spc_base + IPU6_PSYS_REG_SPC_START_PC);  in ipu6_pkg_dir_configure_spc()
262 	writel(pkg_dir_vied_address, base + hw_variant->dmem_offset);  in ipu6_pkg_dir_configure_spc()
270 	void __iomem *dmem_base = base + hw_variant->dmem_offset;  in ipu6_configure_spc()
271 	void __iomem *spc_regs_base = base + hw_variant->spc_offset;  in ipu6_configure_spc()
278 	if (isp->secure_mode)  in ipu6_configure_spc()
293 	u8 hw_ver = isp->hw_ver;  in ipu6_internal_pdata_init()
373 	struct device *dev = &pdev->dev;  in ipu6_isys_init()
386 		return ERR_PTR(-ENOMEM);  in ipu6_isys_init()
388 	pdata->base = base;  in ipu6_isys_init()
389 	pdata->ipdata = ipdata;  in ipu6_isys_init()
399 	isys_adev->mmu = ipu6_mmu_init(dev, base, ISYS_MMID,  in ipu6_isys_init()
400 				       &ipdata->hw_variant);  in ipu6_isys_init()
401 	if (IS_ERR(isys_adev->mmu)) {  in ipu6_isys_init()
402 		put_device(&isys_adev->auxdev.dev);  in ipu6_isys_init()
404 		return dev_err_cast_probe(dev, isys_adev->mmu,  in ipu6_isys_init()
405 				"ipu6_mmu_init(isys_adev->mmu) failed\n");  in ipu6_isys_init()
408 	isys_adev->mmu->dev = &isys_adev->auxdev.dev;  in ipu6_isys_init()
430 		return ERR_PTR(-ENOMEM);  in ipu6_psys_init()
432 	pdata->base = base;  in ipu6_psys_init()
433 	pdata->ipdata = ipdata;  in ipu6_psys_init()
439 		return dev_err_cast_probe(&pdev->dev, psys_adev,  in ipu6_psys_init()
443 	psys_adev->mmu = ipu6_mmu_init(&pdev->dev, base, PSYS_MMID,  in ipu6_psys_init()
444 				       &ipdata->hw_variant);  in ipu6_psys_init()
445 	if (IS_ERR(psys_adev->mmu)) {  in ipu6_psys_init()
446 		put_device(&psys_adev->auxdev.dev);  in ipu6_psys_init()
448 		return dev_err_cast_probe(&pdev->dev, psys_adev->mmu,  in ipu6_psys_init()
449 				"ipu6_mmu_init(psys_adev->mmu) failed\n");  in ipu6_psys_init()
452 	psys_adev->mmu->dev = &psys_adev->auxdev.dev;  in ipu6_psys_init()
481 		return dev_err_probe(&dev->dev, ret, "Request msi failed");  in ipu6_pci_config_setup()
488 	u32 val = readl(isp->base + BUTTRESS_REG_BTRS_CTRL);  in ipu6_configure_vc_mechanism()
500 	writel(val, isp->base + BUTTRESS_REG_BTRS_CTRL);  in ipu6_configure_vc_mechanism()
506 	struct device *dev = &pdev->dev;  in ipu6_pci_probe()
516 		return -ENOMEM;  in ipu6_pci_probe()
518 	isp->pdev = pdev;  in ipu6_pci_probe()
519 	INIT_LIST_HEAD(&isp->devices);  in ipu6_pci_probe()
532 	isp->base = pcim_iomap_table(pdev)[IPU6_PCI_BAR];  in ipu6_pci_probe()
536 	isp->cpd_metadata_cmpnt_size = sizeof(struct ipu6_cpd_metadata_cmpnt);  in ipu6_pci_probe()
537 	switch (id->device) {  in ipu6_pci_probe()
539 		isp->hw_ver = IPU6_VER_6;  in ipu6_pci_probe()
540 		isp->cpd_fw_name = IPU6_FIRMWARE_NAME;  in ipu6_pci_probe()
543 		isp->hw_ver = IPU6_VER_6SE;  in ipu6_pci_probe()
544 		isp->cpd_fw_name = IPU6SE_FIRMWARE_NAME;  in ipu6_pci_probe()
545 		isp->cpd_metadata_cmpnt_size =  in ipu6_pci_probe()
550 		isp->hw_ver = IPU6_VER_6EP;  in ipu6_pci_probe()
551 		isp->cpd_fw_name = IPU6EP_FIRMWARE_NAME;  in ipu6_pci_probe()
554 		isp->hw_ver = IPU6_VER_6EP;  in ipu6_pci_probe()
555 		isp->cpd_fw_name = IPU6EPADLN_FIRMWARE_NAME;  in ipu6_pci_probe()
558 		isp->hw_ver = IPU6_VER_6EP_MTL;  in ipu6_pci_probe()
559 		isp->cpd_fw_name = IPU6EPMTL_FIRMWARE_NAME;  in ipu6_pci_probe()
562 		return dev_err_probe(dev, -ENODEV,  in ipu6_pci_probe()
564 				     id->device);  in ipu6_pci_probe()
569 	isys_base = isp->base + isys_ipdata.hw_variant.offset;  in ipu6_pci_probe()
570 	psys_base = isp->base + psys_ipdata.hw_variant.offset;  in ipu6_pci_probe()
578 	ret = ipu6_pci_config_setup(pdev, isp->hw_ver);  in ipu6_pci_probe()
586 	ret = request_firmware(&isp->cpd_fw, isp->cpd_fw_name, dev);  in ipu6_pci_probe()
588 		dev_err_probe(&isp->pdev->dev, ret,  in ipu6_pci_probe()
590 			      isp->cpd_fw_name);  in ipu6_pci_probe()
594 	ret = ipu6_cpd_validate_cpd_file(isp, isp->cpd_fw->data,  in ipu6_pci_probe()
595 					 isp->cpd_fw->size);  in ipu6_pci_probe()
597 		dev_err_probe(&isp->pdev->dev, ret,  in ipu6_pci_probe()
605 		ret = -ENOMEM;  in ipu6_pci_probe()
609 	isp->isys = ipu6_isys_init(pdev, dev, isys_ctrl, isys_base,  in ipu6_pci_probe()
611 	if (IS_ERR(isp->isys)) {  in ipu6_pci_probe()
612 		ret = PTR_ERR(isp->isys);  in ipu6_pci_probe()
619 		ret = -ENOMEM;  in ipu6_pci_probe()
623 	isp->psys = ipu6_psys_init(pdev, &isp->isys->auxdev.dev, psys_ctrl,  in ipu6_pci_probe()
625 	if (IS_ERR(isp->psys)) {  in ipu6_pci_probe()
626 		ret = PTR_ERR(isp->psys);  in ipu6_pci_probe()
630 	ret = pm_runtime_resume_and_get(&isp->psys->auxdev.dev);  in ipu6_pci_probe()
634 	ret = ipu6_mmu_hw_init(isp->psys->mmu);  in ipu6_pci_probe()
636 		dev_err_probe(&isp->pdev->dev, ret,  in ipu6_pci_probe()
641 	ret = ipu6_buttress_map_fw_image(isp->psys, isp->cpd_fw,  in ipu6_pci_probe()
642 					 &isp->psys->fw_sgt);  in ipu6_pci_probe()
644 		dev_err_probe(&isp->pdev->dev, ret, "failed to map fw image\n");  in ipu6_pci_probe()
648 	ret = ipu6_cpd_create_pkg_dir(isp->psys, isp->cpd_fw->data);  in ipu6_pci_probe()
650 		dev_err_probe(&isp->pdev->dev, ret,  in ipu6_pci_probe()
655 	ret = devm_request_threaded_irq(dev, pdev->irq, ipu6_buttress_isr,  in ipu6_pci_probe()
665 		dev_err_probe(&isp->pdev->dev, ret,  in ipu6_pci_probe()
670 	ipu6_mmu_hw_cleanup(isp->psys->mmu);  in ipu6_pci_probe()
671 	pm_runtime_put(&isp->psys->auxdev.dev);  in ipu6_pci_probe()
676 	val = readl(isp->base + BUTTRESS_REG_SKU);  in ipu6_pci_probe()
679 	dev_info(dev, "IPU%u-v%u[%x] hardware version %d\n", version, sku_id,  in ipu6_pci_probe()
680 		 pdev->device, isp->hw_ver);  in ipu6_pci_probe()
685 	isp->bus_ready_to_probe = true;  in ipu6_pci_probe()
690 	devm_free_irq(dev, pdev->irq, isp);  in ipu6_pci_probe()
692 	if (isp->psys) {  in ipu6_pci_probe()
693 		ipu6_cpd_free_pkg_dir(isp->psys);  in ipu6_pci_probe()
694 		ipu6_buttress_unmap_fw_image(isp->psys, &isp->psys->fw_sgt);  in ipu6_pci_probe()
696 	if (!IS_ERR_OR_NULL(isp->psys) && !IS_ERR_OR_NULL(isp->psys->mmu))  in ipu6_pci_probe()
697 		ipu6_mmu_cleanup(isp->psys->mmu);  in ipu6_pci_probe()
698 	if (!IS_ERR_OR_NULL(isp->isys) && !IS_ERR_OR_NULL(isp->isys->mmu))  in ipu6_pci_probe()
699 		ipu6_mmu_cleanup(isp->isys->mmu);  in ipu6_pci_probe()
701 	release_firmware(isp->cpd_fw);  in ipu6_pci_probe()
711 	struct ipu6_mmu *isys_mmu = isp->isys->mmu;  in ipu6_pci_remove()
712 	struct ipu6_mmu *psys_mmu = isp->psys->mmu;  in ipu6_pci_remove()
714 	devm_free_irq(&pdev->dev, pdev->irq, isp);  in ipu6_pci_remove()
715 	ipu6_cpd_free_pkg_dir(isp->psys);  in ipu6_pci_remove()
717 	ipu6_buttress_unmap_fw_image(isp->psys, &isp->psys->fw_sgt);  in ipu6_pci_remove()
722 	pm_runtime_forbid(&pdev->dev);  in ipu6_pci_remove()
723 	pm_runtime_get_noresume(&pdev->dev);  in ipu6_pci_remove()
725 	release_firmware(isp->cpd_fw);  in ipu6_pci_remove()
735 	pm_runtime_forbid(&isp->pdev->dev);  in ipu6_pci_reset_prepare()
743 	if (isp->secure_mode)  in ipu6_pci_reset_done()
746 	isp->need_ipc_reset = true;  in ipu6_pci_reset_done()
747 	pm_runtime_allow(&isp->pdev->dev);  in ipu6_pci_reset_done()
752  * PCI device level PM state transitions (D0<->D3)
758 	synchronize_irq(pdev->irq);  in ipu6_suspend()
766 	struct ipu6_buttress *b = &isp->buttress;  in ipu6_resume()
772 	isp->secure_mode = ipu6_buttress_get_secure_mode(isp);  in ipu6_resume()
774 		 isp->secure_mode ? "secure" : "non-secure");  in ipu6_resume()
778 	ret = ipu6_buttress_ipc_reset(isp, &b->cse);  in ipu6_resume()
780 		dev_err(&isp->pdev->dev, "IPC reset protocol failed!\n");  in ipu6_resume()
782 	ret = pm_runtime_resume_and_get(&isp->psys->auxdev.dev);  in ipu6_resume()
784 		dev_err(&isp->psys->auxdev.dev, "Failed to get runtime PM\n");  in ipu6_resume()
790 		dev_err(&isp->pdev->dev, "FW authentication failed(%d)\n", ret);  in ipu6_resume()
792 	pm_runtime_put(&isp->psys->auxdev.dev);  in ipu6_resume()
806 	if (isp->need_ipc_reset) {  in ipu6_runtime_resume()
807 		struct ipu6_buttress *b = &isp->buttress;  in ipu6_runtime_resume()
809 		isp->need_ipc_reset = false;  in ipu6_runtime_resume()
810 		ret = ipu6_buttress_ipc_reset(isp, &b->cse);  in ipu6_runtime_resume()
812 			dev_err(&isp->pdev->dev, "IPC reset protocol failed\n");  in ipu6_runtime_resume()