71 	struct ipmmu_vmsa_device *mmu;  member
99 /* MMU "context" registers */
149 static bool ipmmu_is_root(struct ipmmu_vmsa_device *mmu)  in ipmmu_is_root()  argument
151 	return mmu->root == mmu;  in ipmmu_is_root()
156 	struct ipmmu_vmsa_device *mmu = dev_get_drvdata(dev);  in __ipmmu_check_device()  local
159 	if (ipmmu_is_root(mmu))  in __ipmmu_check_device()
160 		*rootp = mmu;  in __ipmmu_check_device()
177 static u32 ipmmu_read(struct ipmmu_vmsa_device *mmu, unsigned int offset)  in ipmmu_read()  argument
179 	return ioread32(mmu->base + offset);  in ipmmu_read()
182 static void ipmmu_write(struct ipmmu_vmsa_device *mmu, unsigned int offset,  in ipmmu_write()  argument
185 	iowrite32(data, mmu->base + offset);  in ipmmu_write()
188 static unsigned int ipmmu_ctx_reg(struct ipmmu_vmsa_device *mmu,  in ipmmu_ctx_reg()  argument
191 	unsigned int base = mmu->features->ctx_offset_base;  in ipmmu_ctx_reg()
196 	return base + context_id * mmu->features->ctx_offset_stride + reg;  in ipmmu_ctx_reg()
199 static u32 ipmmu_ctx_read(struct ipmmu_vmsa_device *mmu,  in ipmmu_ctx_read()  argument
202 	return ipmmu_read(mmu, ipmmu_ctx_reg(mmu, context_id, reg));  in ipmmu_ctx_read()
205 static void ipmmu_ctx_write(struct ipmmu_vmsa_device *mmu,  in ipmmu_ctx_write()  argument
208 	ipmmu_write(mmu, ipmmu_ctx_reg(mmu, context_id, reg), data);  in ipmmu_ctx_write()
214 	return ipmmu_ctx_read(domain->mmu->root, domain->context_id, reg);  in ipmmu_ctx_read_root()
220 	ipmmu_ctx_write(domain->mmu->root, domain->context_id, reg, data);  in ipmmu_ctx_write_root()
226 	if (domain->mmu != domain->mmu->root)  in ipmmu_ctx_write_all()
227 		ipmmu_ctx_write(domain->mmu, domain->context_id, reg, data);  in ipmmu_ctx_write_all()
229 	ipmmu_ctx_write(domain->mmu->root, domain->context_id, reg, data);  in ipmmu_ctx_write_all()
232 static u32 ipmmu_utlb_reg(struct ipmmu_vmsa_device *mmu, unsigned int reg)  in ipmmu_utlb_reg()  argument
234 	return mmu->features->utlb_offset_base + reg;  in ipmmu_utlb_reg()
237 static void ipmmu_imuasid_write(struct ipmmu_vmsa_device *mmu,  in ipmmu_imuasid_write()  argument
240 	ipmmu_write(mmu, ipmmu_utlb_reg(mmu, IMUASID(utlb)), data);  in ipmmu_imuasid_write()
243 static void ipmmu_imuctr_write(struct ipmmu_vmsa_device *mmu,  in ipmmu_imuctr_write()  argument
246 	ipmmu_write(mmu, ipmmu_utlb_reg(mmu, IMUCTR(utlb)), data);  in ipmmu_imuctr_write()
261 		dev_err_ratelimited(domain->mmu->dev,  in ipmmu_tlb_sync()
262 			"TLB sync timed out -- MMU may be deadlocked\n");  in ipmmu_tlb_sync()
277  * Enable MMU translation for the microTLB.
282 	struct ipmmu_vmsa_device *mmu = domain->mmu;  in ipmmu_utlb_enable()  local
290 	ipmmu_imuasid_write(mmu, utlb, 0);  in ipmmu_utlb_enable()
292 	ipmmu_imuctr_write(mmu, utlb, IMUCTR_TTSEL_MMU(domain->context_id) |  in ipmmu_utlb_enable()
294 	mmu->utlb_ctx[utlb] = domain->context_id;  in ipmmu_utlb_enable()
298  * Disable MMU translation for the microTLB.
303 	struct ipmmu_vmsa_device *mmu = domain->mmu;  in ipmmu_utlb_disable()  local
305 	ipmmu_imuctr_write(mmu, utlb, 0);  in ipmmu_utlb_disable()
306 	mmu->utlb_ctx[utlb] = IPMMU_CTX_INVALID;  in ipmmu_utlb_disable()
331 static int ipmmu_domain_allocate_context(struct ipmmu_vmsa_device *mmu,  in ipmmu_domain_allocate_context()  argument
337 	spin_lock_irqsave(&mmu->lock, flags);  in ipmmu_domain_allocate_context()
339 	ret = find_first_zero_bit(mmu->ctx, mmu->num_ctx);  in ipmmu_domain_allocate_context()
340 	if (ret != mmu->num_ctx) {  in ipmmu_domain_allocate_context()
341 		mmu->domains[ret] = domain;  in ipmmu_domain_allocate_context()
342 		set_bit(ret, mmu->ctx);  in ipmmu_domain_allocate_context()
346 	spin_unlock_irqrestore(&mmu->lock, flags);  in ipmmu_domain_allocate_context()
351 static void ipmmu_domain_free_context(struct ipmmu_vmsa_device *mmu,  in ipmmu_domain_free_context()  argument
356 	spin_lock_irqsave(&mmu->lock, flags);  in ipmmu_domain_free_context()
358 	clear_bit(context_id, mmu->ctx);  in ipmmu_domain_free_context()
359 	mmu->domains[context_id] = NULL;  in ipmmu_domain_free_context()
361 	spin_unlock_irqrestore(&mmu->lock, flags);  in ipmmu_domain_free_context()
379 	if (domain->mmu->features->twobit_imttbcr_sl0)  in ipmmu_domain_setup_context()
384 	if (domain->mmu->features->cache_snoop)  in ipmmu_domain_setup_context()
395 	if (domain->mmu->features->setup_imbuscr)  in ipmmu_domain_setup_context()
408 	 * Enable the MMU and interrupt generation. The long-descriptor  in ipmmu_domain_setup_context()
444 	domain->cfg.iommu_dev = domain->mmu->root->dev;  in ipmmu_domain_init_context()
449 	ret = ipmmu_domain_allocate_context(domain->mmu->root, domain);  in ipmmu_domain_init_context()
458 		ipmmu_domain_free_context(domain->mmu->root,  in ipmmu_domain_init_context()
469 	if (!domain->mmu)  in ipmmu_domain_destroy_context()
480 	ipmmu_domain_free_context(domain->mmu->root, domain->context_id);  in ipmmu_domain_destroy_context()
490 	struct ipmmu_vmsa_device *mmu = domain->mmu;  in ipmmu_domain_irq()  local
512 		dev_err_ratelimited(mmu->dev, "Multiple TLB hits @0x%lx\n",  in ipmmu_domain_irq()
515 		dev_err_ratelimited(mmu->dev, "Page Table Walk Abort @0x%lx\n",  in ipmmu_domain_irq()
527 	if (!report_iommu_fault(&domain->io_domain, mmu->dev, iova, 0))  in ipmmu_domain_irq()
530 	dev_err_ratelimited(mmu->dev,  in ipmmu_domain_irq()
539 	struct ipmmu_vmsa_device *mmu = dev;  in ipmmu_irq()  local
544 	spin_lock_irqsave(&mmu->lock, flags);  in ipmmu_irq()
549 	for (i = 0; i < mmu->num_ctx; i++) {  in ipmmu_irq()
550 		if (!mmu->domains[i])  in ipmmu_irq()
552 		if (ipmmu_domain_irq(mmu->domains[i]) == IRQ_HANDLED)  in ipmmu_irq()
556 	spin_unlock_irqrestore(&mmu->lock, flags);  in ipmmu_irq()
595 	struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);  in ipmmu_attach_device()  local
600 	if (!mmu) {  in ipmmu_attach_device()
607 	if (!domain->mmu) {  in ipmmu_attach_device()
609 		domain->mmu = mmu;  in ipmmu_attach_device()
613 			domain->mmu = NULL;  in ipmmu_attach_device()
618 	} else if (domain->mmu != mmu) {  in ipmmu_attach_device()
691 	if (domain->mmu)  in ipmmu_flush_iotlb_all()
792 	struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);  in ipmmu_init_arm_mapping()  local
804 	if (!mmu->mapping) {  in ipmmu_init_arm_mapping()
809 			dev_err(mmu->dev, "failed to create ARM IOMMU mapping\n");  in ipmmu_init_arm_mapping()
814 		mmu->mapping = mapping;  in ipmmu_init_arm_mapping()
818 	ret = arm_iommu_attach_device(dev, mmu->mapping);  in ipmmu_init_arm_mapping()
827 	if (mmu->mapping)  in ipmmu_init_arm_mapping()
828 		arm_iommu_release_mapping(mmu->mapping);  in ipmmu_init_arm_mapping()
835 	struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);  in ipmmu_probe_device()  local
840 	if (!mmu)  in ipmmu_probe_device()
843 	return &mmu->iommu;  in ipmmu_probe_device()
860 	struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);  in ipmmu_release_device()  local
866 		ipmmu_imuctr_write(mmu, utlb, 0);  in ipmmu_release_device()
867 		mmu->utlb_ctx[utlb] = IPMMU_CTX_INVALID;  in ipmmu_release_device()
870 	arm_iommu_release_mapping(mmu->mapping);  in ipmmu_release_device()
902 static void ipmmu_device_reset(struct ipmmu_vmsa_device *mmu)  in ipmmu_device_reset()  argument
907 	for (i = 0; i < mmu->num_ctx; ++i)  in ipmmu_device_reset()
908 		ipmmu_ctx_write(mmu, i, IMCTR, 0);  in ipmmu_device_reset()
1006 	struct ipmmu_vmsa_device *mmu;  in ipmmu_probe()  local
1010 	mmu = devm_kzalloc(&pdev->dev, sizeof(*mmu), GFP_KERNEL);  in ipmmu_probe()
1011 	if (!mmu) {  in ipmmu_probe()
1016 	mmu->dev = &pdev->dev;  in ipmmu_probe()
1017 	spin_lock_init(&mmu->lock);  in ipmmu_probe()
1018 	bitmap_zero(mmu->ctx, IPMMU_CTX_MAX);  in ipmmu_probe()
1019 	mmu->features = of_device_get_match_data(&pdev->dev);  in ipmmu_probe()
1020 	memset(mmu->utlb_ctx, IPMMU_CTX_INVALID, mmu->features->num_utlbs);  in ipmmu_probe()
1026 	mmu->base = devm_platform_ioremap_resource(pdev, 0);  in ipmmu_probe()
1027 	if (IS_ERR(mmu->base))  in ipmmu_probe()
1028 		return PTR_ERR(mmu->base);  in ipmmu_probe()
1042 	if (mmu->features->use_ns_alias_offset)  in ipmmu_probe()
1043 		mmu->base += IM_NS_ALIAS_OFFSET;  in ipmmu_probe()
1045 	mmu->num_ctx = min(IPMMU_CTX_MAX, mmu->features->number_of_contexts);  in ipmmu_probe()
1051 	if (!mmu->features->has_cache_leaf_nodes ||  in ipmmu_probe()
1053 		mmu->root = mmu;  in ipmmu_probe()
1055 		mmu->root = ipmmu_find_root();  in ipmmu_probe()
1060 	if (!mmu->root)  in ipmmu_probe()
1064 	if (ipmmu_is_root(mmu)) {  in ipmmu_probe()
1070 				       dev_name(&pdev->dev), mmu);  in ipmmu_probe()
1076 		ipmmu_device_reset(mmu);  in ipmmu_probe()
1078 		if (mmu->features->reserved_context) {  in ipmmu_probe()
1080 			set_bit(0, mmu->ctx);  in ipmmu_probe()
1089 	if (!mmu->features->has_cache_leaf_nodes || !ipmmu_is_root(mmu)) {  in ipmmu_probe()
1090 		ret = iommu_device_sysfs_add(&mmu->iommu, &pdev->dev, NULL,  in ipmmu_probe()
1095 		ret = iommu_device_register(&mmu->iommu, &ipmmu_ops, &pdev->dev);  in ipmmu_probe()
1106 	platform_set_drvdata(pdev, mmu);  in ipmmu_probe()
1113 	struct ipmmu_vmsa_device *mmu = platform_get_drvdata(pdev);  in ipmmu_remove()  local
1115 	iommu_device_sysfs_remove(&mmu->iommu);  in ipmmu_remove()
1116 	iommu_device_unregister(&mmu->iommu);  in ipmmu_remove()
1118 	arm_iommu_release_mapping(mmu->mapping);  in ipmmu_remove()
1120 	ipmmu_device_reset(mmu);  in ipmmu_remove()
1125 	struct ipmmu_vmsa_device *mmu = dev_get_drvdata(dev);  in ipmmu_resume_noirq()  local
1128 	/* Reset root MMU and restore contexts */  in ipmmu_resume_noirq()
1129 	if (ipmmu_is_root(mmu)) {  in ipmmu_resume_noirq()
1130 		ipmmu_device_reset(mmu);  in ipmmu_resume_noirq()
1132 		for (i = 0; i < mmu->num_ctx; i++) {  in ipmmu_resume_noirq()
1133 			if (!mmu->domains[i])  in ipmmu_resume_noirq()
1136 			ipmmu_domain_setup_context(mmu->domains[i]);  in ipmmu_resume_noirq()
1141 	for (i = 0; i < mmu->features->num_utlbs; i++) {  in ipmmu_resume_noirq()
1142 		if (mmu->utlb_ctx[i] == IPMMU_CTX_INVALID)  in ipmmu_resume_noirq()
1145 		ipmmu_utlb_enable(mmu->root->domains[mmu->utlb_ctx[i]], i);  in ipmmu_resume_noirq()