1 // SPDX-License-Identifier: GPL-2.0-only
3  * IOMMU API for QCOM secure IOMMUs.  Somewhat based on arm-smmu.c
13 #include <linux/dma-mapping.h>
17 #include <linux/io-64-nonatomic-hi-lo.h>
18 #include <linux/io-pgtable.h>
33 #include "arm-smmu.h"
54 	struct qcom_iommu_ctx	*ctxs[];   /* indexed by asid */
62 	u8			 asid;      /* asid and ctx bank # are 1:1 */  member
82 static struct qcom_iommu_ctx * to_ctx(struct qcom_iommu_domain *d, unsigned asid)  in to_ctx()  argument
84 	struct qcom_iommu_dev *qcom_iommu = d->iommu;  in to_ctx()
87 	return qcom_iommu->ctxs[asid];  in to_ctx()
91 iommu_writel(struct qcom_iommu_ctx *ctx, unsigned reg, u32 val)  in iommu_writel()  argument
93 	writel_relaxed(val, ctx->base + reg);  in iommu_writel()
97 iommu_writeq(struct qcom_iommu_ctx *ctx, unsigned reg, u64 val)  in iommu_writeq()  argument
99 	writeq_relaxed(val, ctx->base + reg);  in iommu_writeq()
103 iommu_readl(struct qcom_iommu_ctx *ctx, unsigned reg)  in iommu_readl()  argument
105 	return readl_relaxed(ctx->base + reg);  in iommu_readl()
109 iommu_readq(struct qcom_iommu_ctx *ctx, unsigned reg)  in iommu_readq()  argument
111 	return readq_relaxed(ctx->base + reg);  in iommu_readq()
117 	struct iommu_fwspec *fwspec = qcom_domain->fwspec;  in qcom_iommu_tlb_sync()
120 	for (i = 0; i < fwspec->num_ids; i++) {  in qcom_iommu_tlb_sync()
121 		struct qcom_iommu_ctx *ctx = to_ctx(qcom_domain, fwspec->ids[i]);  in qcom_iommu_tlb_sync()  local
124 		iommu_writel(ctx, ARM_SMMU_CB_TLBSYNC, 0);  in qcom_iommu_tlb_sync()
126 		ret = readl_poll_timeout(ctx->base + ARM_SMMU_CB_TLBSTATUS, val,  in qcom_iommu_tlb_sync()
129 			dev_err(ctx->dev, "timeout waiting for TLB SYNC\n");  in qcom_iommu_tlb_sync()
136 	struct iommu_fwspec *fwspec = qcom_domain->fwspec;  in qcom_iommu_tlb_inv_context()
139 	for (i = 0; i < fwspec->num_ids; i++) {  in qcom_iommu_tlb_inv_context()
140 		struct qcom_iommu_ctx *ctx = to_ctx(qcom_domain, fwspec->ids[i]);  in qcom_iommu_tlb_inv_context()  local
141 		iommu_writel(ctx, ARM_SMMU_CB_S1_TLBIASID, ctx->asid);  in qcom_iommu_tlb_inv_context()
151 	struct iommu_fwspec *fwspec = qcom_domain->fwspec;  in qcom_iommu_tlb_inv_range_nosync()
156 	for (i = 0; i < fwspec->num_ids; i++) {  in qcom_iommu_tlb_inv_range_nosync()
157 		struct qcom_iommu_ctx *ctx = to_ctx(qcom_domain, fwspec->ids[i]);  in qcom_iommu_tlb_inv_range_nosync()  local
161 		iova |= ctx->asid;  in qcom_iommu_tlb_inv_range_nosync()
163 			iommu_writel(ctx, reg, iova);  in qcom_iommu_tlb_inv_range_nosync()
165 		} while (s -= granule);  in qcom_iommu_tlb_inv_range_nosync()
191 	struct qcom_iommu_ctx *ctx = dev;  in qcom_iommu_fault()  local
195 	fsr = iommu_readl(ctx, ARM_SMMU_CB_FSR);  in qcom_iommu_fault()
200 	fsynr = iommu_readl(ctx, ARM_SMMU_CB_FSYNR0);  in qcom_iommu_fault()
201 	iova = iommu_readq(ctx, ARM_SMMU_CB_FAR);  in qcom_iommu_fault()
203 	if (!report_iommu_fault(ctx->domain, ctx->dev, iova, 0)) {  in qcom_iommu_fault()
204 		dev_err_ratelimited(ctx->dev,  in qcom_iommu_fault()
207 				    fsr, iova, fsynr, ctx->asid);  in qcom_iommu_fault()
210 	iommu_writel(ctx, ARM_SMMU_CB_FSR, fsr);  in qcom_iommu_fault()
211 	iommu_writel(ctx, ARM_SMMU_CB_RESUME, ARM_SMMU_RESUME_TERMINATE);  in qcom_iommu_fault()
227 	mutex_lock(&qcom_domain->init_mutex);  in qcom_iommu_init_domain()
228 	if (qcom_domain->iommu)  in qcom_iommu_init_domain()
236 		.iommu_dev	= qcom_iommu->dev,  in qcom_iommu_init_domain()
239 	qcom_domain->iommu = qcom_iommu;  in qcom_iommu_init_domain()
240 	qcom_domain->fwspec = fwspec;  in qcom_iommu_init_domain()
244 		dev_err(qcom_iommu->dev, "failed to allocate pagetable ops\n");  in qcom_iommu_init_domain()
245 		ret = -ENOMEM;  in qcom_iommu_init_domain()
250 	domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;  in qcom_iommu_init_domain()
251 	domain->geometry.aperture_end = (1ULL << pgtbl_cfg.ias) - 1;  in qcom_iommu_init_domain()
252 	domain->geometry.force_aperture = true;  in qcom_iommu_init_domain()
254 	for (i = 0; i < fwspec->num_ids; i++) {  in qcom_iommu_init_domain()
255 		struct qcom_iommu_ctx *ctx = to_ctx(qcom_domain, fwspec->ids[i]);  in qcom_iommu_init_domain()  local
257 		if (!ctx->secure_init) {  in qcom_iommu_init_domain()
258 			ret = qcom_scm_restore_sec_cfg(qcom_iommu->sec_id, ctx->asid);  in qcom_iommu_init_domain()
260 				dev_err(qcom_iommu->dev, "secure init failed: %d\n", ret);  in qcom_iommu_init_domain()
263 			ctx->secure_init = true;  in qcom_iommu_init_domain()
266 		/* Secured QSMMU-500/QSMMU-v2 contexts cannot be programmed */  in qcom_iommu_init_domain()
267 		if (ctx->secured_ctx) {  in qcom_iommu_init_domain()
268 			ctx->domain = domain;  in qcom_iommu_init_domain()
273 		iommu_writel(ctx, ARM_SMMU_CB_SCTLR, 0);  in qcom_iommu_init_domain()
276 		iommu_writel(ctx, ARM_SMMU_CB_FAR, 0);  in qcom_iommu_init_domain()
277 		iommu_writel(ctx, ARM_SMMU_CB_FSR, ARM_SMMU_CB_FSR_FAULT);  in qcom_iommu_init_domain()
280 		iommu_writeq(ctx, ARM_SMMU_CB_TTBR0,  in qcom_iommu_init_domain()
282 				FIELD_PREP(ARM_SMMU_TTBRn_ASID, ctx->asid));  in qcom_iommu_init_domain()
283 		iommu_writeq(ctx, ARM_SMMU_CB_TTBR1, 0);  in qcom_iommu_init_domain()
286 		iommu_writel(ctx, ARM_SMMU_CB_TCR2,  in qcom_iommu_init_domain()
288 		iommu_writel(ctx, ARM_SMMU_CB_TCR,  in qcom_iommu_init_domain()
291 		/* MAIRs (stage-1 only) */  in qcom_iommu_init_domain()
292 		iommu_writel(ctx, ARM_SMMU_CB_S1_MAIR0,  in qcom_iommu_init_domain()
294 		iommu_writel(ctx, ARM_SMMU_CB_S1_MAIR1,  in qcom_iommu_init_domain()
306 		iommu_writel(ctx, ARM_SMMU_CB_SCTLR, reg);  in qcom_iommu_init_domain()
308 		ctx->domain = domain;  in qcom_iommu_init_domain()
311 	mutex_unlock(&qcom_domain->init_mutex);  in qcom_iommu_init_domain()
314 	qcom_domain->pgtbl_ops = pgtbl_ops;  in qcom_iommu_init_domain()
319 	qcom_domain->iommu = NULL;  in qcom_iommu_init_domain()
321 	mutex_unlock(&qcom_domain->init_mutex);  in qcom_iommu_init_domain()
338 	mutex_init(&qcom_domain->init_mutex);  in qcom_iommu_domain_alloc_paging()
339 	spin_lock_init(&qcom_domain->pgtbl_lock);  in qcom_iommu_domain_alloc_paging()
341 	return &qcom_domain->domain;  in qcom_iommu_domain_alloc_paging()
348 	if (qcom_domain->iommu) {  in qcom_iommu_domain_free()
351 		 * off, for example, with GPUs or anything involving dma-buf.  in qcom_iommu_domain_free()
355 		pm_runtime_get_sync(qcom_domain->iommu->dev);  in qcom_iommu_domain_free()
356 		free_io_pgtable_ops(qcom_domain->pgtbl_ops);  in qcom_iommu_domain_free()
357 		pm_runtime_put_sync(qcom_domain->iommu->dev);  in qcom_iommu_domain_free()
371 		return -ENXIO;  in qcom_iommu_attach_dev()
375 	pm_runtime_get_sync(qcom_iommu->dev);  in qcom_iommu_attach_dev()
377 	pm_runtime_put_sync(qcom_iommu->dev);  in qcom_iommu_attach_dev()
385 	if (qcom_domain->iommu != qcom_iommu)  in qcom_iommu_attach_dev()
386 		return -EINVAL;  in qcom_iommu_attach_dev()
404 	if (WARN_ON(!qcom_domain->iommu))  in qcom_iommu_identity_attach()
405 		return -EINVAL;  in qcom_iommu_identity_attach()
407 	pm_runtime_get_sync(qcom_iommu->dev);  in qcom_iommu_identity_attach()
408 	for (i = 0; i < fwspec->num_ids; i++) {  in qcom_iommu_identity_attach()
409 		struct qcom_iommu_ctx *ctx = to_ctx(qcom_domain, fwspec->ids[i]);  in qcom_iommu_identity_attach()  local
412 		iommu_writel(ctx, ARM_SMMU_CB_SCTLR, 0);  in qcom_iommu_identity_attach()
414 		ctx->domain = NULL;  in qcom_iommu_identity_attach()
416 	pm_runtime_put_sync(qcom_iommu->dev);  in qcom_iommu_identity_attach()
436 	struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;  in qcom_iommu_map()
439 		return -ENODEV;  in qcom_iommu_map()
441 	spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);  in qcom_iommu_map()
442 	ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, GFP_ATOMIC, mapped);  in qcom_iommu_map()
443 	spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);  in qcom_iommu_map()
454 	struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;  in qcom_iommu_unmap()
460 	 * for example, with GPUs or anything involving dma-buf.  So we  in qcom_iommu_unmap()
464 	pm_runtime_get_sync(qcom_domain->iommu->dev);  in qcom_iommu_unmap()
465 	spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);  in qcom_iommu_unmap()
466 	ret = ops->unmap_pages(ops, iova, pgsize, pgcount, gather);  in qcom_iommu_unmap()
467 	spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);  in qcom_iommu_unmap()
468 	pm_runtime_put_sync(qcom_domain->iommu->dev);  in qcom_iommu_unmap()
476 	struct io_pgtable *pgtable = container_of(qcom_domain->pgtbl_ops,  in qcom_iommu_flush_iotlb_all()
478 	if (!qcom_domain->pgtbl_ops)  in qcom_iommu_flush_iotlb_all()
481 	pm_runtime_get_sync(qcom_domain->iommu->dev);  in qcom_iommu_flush_iotlb_all()
482 	qcom_iommu_tlb_sync(pgtable->cookie);  in qcom_iommu_flush_iotlb_all()
483 	pm_runtime_put_sync(qcom_domain->iommu->dev);  in qcom_iommu_flush_iotlb_all()
498 	struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;  in qcom_iommu_iova_to_phys()
503 	spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);  in qcom_iommu_iova_to_phys()
504 	ret = ops->iova_to_phys(ops, iova);  in qcom_iommu_iova_to_phys()
505 	spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);  in qcom_iommu_iova_to_phys()
532 		return ERR_PTR(-ENODEV);  in qcom_iommu_probe_device()
539 	link = device_link_add(dev, qcom_iommu->dev, DL_FLAG_PM_RUNTIME);  in qcom_iommu_probe_device()
541 		dev_err(qcom_iommu->dev, "Unable to create device link between %s and %s\n",  in qcom_iommu_probe_device()
542 			dev_name(qcom_iommu->dev), dev_name(dev));  in qcom_iommu_probe_device()
543 		return ERR_PTR(-ENODEV);  in qcom_iommu_probe_device()
546 	return &qcom_iommu->iommu;  in qcom_iommu_probe_device()
554 	unsigned asid = args->args[0];  in qcom_iommu_of_xlate()  local
556 	if (args->args_count != 1) {  in qcom_iommu_of_xlate()
559 			args->np->full_name, args->args_count);  in qcom_iommu_of_xlate()
560 		return -EINVAL;  in qcom_iommu_of_xlate()
563 	iommu_pdev = of_find_device_by_node(args->np);  in qcom_iommu_of_xlate()
565 		return -EINVAL;  in qcom_iommu_of_xlate()
569 	/* make sure the asid specified in dt is valid, so we don't have  in qcom_iommu_of_xlate()
572 	if (WARN_ON(asid > qcom_iommu->max_asid) ||  in qcom_iommu_of_xlate()
573 	    WARN_ON(qcom_iommu->ctxs[asid] == NULL)) {  in qcom_iommu_of_xlate()
574 		put_device(&iommu_pdev->dev);  in qcom_iommu_of_xlate()
575 		return -EINVAL;  in qcom_iommu_of_xlate()
586 			put_device(&iommu_pdev->dev);  in qcom_iommu_of_xlate()
587 			return -EINVAL;  in qcom_iommu_of_xlate()
591 	return iommu_fwspec_add_ids(dev, &asid, 1);  in qcom_iommu_of_xlate()
641 		return -ENOMEM;  in qcom_iommu_sec_ptbl_init()
661 	int asid;  in get_asid()  local
664 	 * of the context bank, and calculate the asid from that:  in get_asid()
667 		return -ENODEV;  in get_asid()
670 	 * Context banks are 0x1000 apart but, in some cases, the ASID  in get_asid()
674 	if (!of_property_read_u32(np, "qcom,ctx-asid", &val))  in get_asid()
675 		asid = val;  in get_asid()
677 		asid = reg / 0x1000;  in get_asid()
679 	return asid;  in get_asid()
684 	struct qcom_iommu_ctx *ctx;  in qcom_iommu_ctx_probe()  local
685 	struct device *dev = &pdev->dev;  in qcom_iommu_ctx_probe()
686 	struct qcom_iommu_dev *qcom_iommu = dev_get_drvdata(dev->parent);  in qcom_iommu_ctx_probe()
689 	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);  in qcom_iommu_ctx_probe()
690 	if (!ctx)  in qcom_iommu_ctx_probe()
691 		return -ENOMEM;  in qcom_iommu_ctx_probe()
693 	ctx->dev = dev;  in qcom_iommu_ctx_probe()
694 	platform_set_drvdata(pdev, ctx);  in qcom_iommu_ctx_probe()
696 	ctx->base = devm_platform_ioremap_resource(pdev, 0);  in qcom_iommu_ctx_probe()
697 	if (IS_ERR(ctx->base))  in qcom_iommu_ctx_probe()
698 		return PTR_ERR(ctx->base);  in qcom_iommu_ctx_probe()
704 	if (of_device_is_compatible(dev->of_node, "qcom,msm-iommu-v2-sec"))  in qcom_iommu_ctx_probe()
705 		ctx->secured_ctx = true;  in qcom_iommu_ctx_probe()
708 	 * boot-loader left us a surprise:  in qcom_iommu_ctx_probe()
710 	if (!ctx->secured_ctx)  in qcom_iommu_ctx_probe()
711 		iommu_writel(ctx, ARM_SMMU_CB_FSR, iommu_readl(ctx, ARM_SMMU_CB_FSR));  in qcom_iommu_ctx_probe()
716 			       "qcom-iommu-fault",  in qcom_iommu_ctx_probe()
717 			       ctx);  in qcom_iommu_ctx_probe()
723 	ret = get_asid(dev->of_node);  in qcom_iommu_ctx_probe()
729 	ctx->asid = ret;  in qcom_iommu_ctx_probe()
731 	dev_dbg(dev, "found asid %u\n", ctx->asid);  in qcom_iommu_ctx_probe()
733 	qcom_iommu->ctxs[ctx->asid] = ctx;  in qcom_iommu_ctx_probe()
740 	struct qcom_iommu_dev *qcom_iommu = dev_get_drvdata(pdev->dev.parent);  in qcom_iommu_ctx_remove()
741 	struct qcom_iommu_ctx *ctx = platform_get_drvdata(pdev);  in qcom_iommu_ctx_remove()  local
745 	qcom_iommu->ctxs[ctx->asid] = NULL;  in qcom_iommu_ctx_remove()
749 	{ .compatible = "qcom,msm-iommu-v1-ns" },
750 	{ .compatible = "qcom,msm-iommu-v1-sec" },
751 	{ .compatible = "qcom,msm-iommu-v2-ns" },
752 	{ .compatible = "qcom,msm-iommu-v2-sec" },
758 		.name		= "qcom-iommu-ctx",
769 	for_each_child_of_node(qcom_iommu->dev->of_node, child) {  in qcom_iommu_has_secure_context()
770 		if (of_device_is_compatible(child, "qcom,msm-iommu-v1-sec") ||  in qcom_iommu_has_secure_context()
771 		    of_device_is_compatible(child, "qcom,msm-iommu-v2-sec")) {  in qcom_iommu_has_secure_context()
784 	struct device *dev = &pdev->dev;  in qcom_iommu_device_probe()
789 	/* find the max asid (which is 1:1 to ctx bank idx), so we know how  in qcom_iommu_device_probe()
790 	 * many child ctx devices we have:  in qcom_iommu_device_probe()
792 	for_each_child_of_node(dev->of_node, child)  in qcom_iommu_device_probe()
798 		return -ENOMEM;  in qcom_iommu_device_probe()
799 	qcom_iommu->max_asid = max_asid;  in qcom_iommu_device_probe()
800 	qcom_iommu->dev = dev;  in qcom_iommu_device_probe()
804 		qcom_iommu->local_base = devm_ioremap_resource(dev, res);  in qcom_iommu_device_probe()
805 		if (IS_ERR(qcom_iommu->local_base))  in qcom_iommu_device_probe()
806 			return PTR_ERR(qcom_iommu->local_base);  in qcom_iommu_device_probe()
814 	qcom_iommu->clks[CLK_IFACE].clk = clk;  in qcom_iommu_device_probe()
821 	qcom_iommu->clks[CLK_BUS].clk = clk;  in qcom_iommu_device_probe()
828 	qcom_iommu->clks[CLK_TBU].clk = clk;  in qcom_iommu_device_probe()
830 	if (of_property_read_u32(dev->of_node, "qcom,iommu-secure-id",  in qcom_iommu_device_probe()
831 				 &qcom_iommu->sec_id)) {  in qcom_iommu_device_probe()
832 		dev_err(dev, "missing qcom,iommu-secure-id property\n");  in qcom_iommu_device_probe()
833 		return -ENODEV;  in qcom_iommu_device_probe()
855 	ret = iommu_device_sysfs_add(&qcom_iommu->iommu, dev, NULL,  in qcom_iommu_device_probe()
862 	ret = iommu_device_register(&qcom_iommu->iommu, &qcom_iommu_ops, dev);  in qcom_iommu_device_probe()
868 	if (qcom_iommu->local_base) {  in qcom_iommu_device_probe()
870 		writel_relaxed(0xffffffff, qcom_iommu->local_base + SMMU_INTR_SEL_NS);  in qcom_iommu_device_probe()
885 	pm_runtime_force_suspend(&pdev->dev);  in qcom_iommu_device_remove()
887 	iommu_device_sysfs_remove(&qcom_iommu->iommu);  in qcom_iommu_device_remove()
888 	iommu_device_unregister(&qcom_iommu->iommu);  in qcom_iommu_device_remove()
896 	ret = clk_bulk_prepare_enable(CLK_NUM, qcom_iommu->clks);  in qcom_iommu_resume()
900 	if (dev->pm_domain)  in qcom_iommu_resume()
901 		return qcom_scm_restore_sec_cfg(qcom_iommu->sec_id, 0);  in qcom_iommu_resume()
910 	clk_bulk_disable_unprepare(CLK_NUM, qcom_iommu->clks);  in qcom_iommu_suspend()
922 	{ .compatible = "qcom,msm-iommu-v1" },
923 	{ .compatible = "qcom,msm-iommu-v2" },
929 		.name		= "qcom-iommu",