29 #include "ipu6-mmu.h"
54 static void tlb_invalidate(struct ipu6_mmu *mmu)  in tlb_invalidate()  argument
59 	spin_lock_irqsave(&mmu->ready_lock, flags);  in tlb_invalidate()
60 	if (!mmu->ready) {  in tlb_invalidate()
61 		spin_unlock_irqrestore(&mmu->ready_lock, flags);  in tlb_invalidate()
65 	for (i = 0; i < mmu->nr_mmus; i++) {  in tlb_invalidate()
74 		if (mmu->mmu_hw[i].insert_read_before_invalidate)  in tlb_invalidate()
75 			readl(mmu->mmu_hw[i].base + REG_L1_PHYS);  in tlb_invalidate()
77 		writel(0xffffffff, mmu->mmu_hw[i].base +  in tlb_invalidate()
87 	spin_unlock_irqrestore(&mmu->ready_lock, flags);  in tlb_invalidate()
414 static int allocate_trash_buffer(struct ipu6_mmu *mmu)  in allocate_trash_buffer()  argument
424 	iova = alloc_iova(&mmu->dmap->iovad, n_pages,  in allocate_trash_buffer()
425 			  PHYS_PFN(mmu->dmap->mmu_info->aperture_end), 0);  in allocate_trash_buffer()
427 		dev_err(mmu->dev, "cannot allocate iova range for trash\n");  in allocate_trash_buffer()
431 	dma = dma_map_page(mmu->dmap->mmu_info->dev, mmu->trash_page, 0,  in allocate_trash_buffer()
433 	if (dma_mapping_error(mmu->dmap->mmu_info->dev, dma)) {  in allocate_trash_buffer()
434 		dev_err(mmu->dmap->mmu_info->dev, "Failed to map trash page\n");  in allocate_trash_buffer()
439 	mmu->pci_trash_page = dma;  in allocate_trash_buffer()
443 	 * mmu->trash_page which is already reserved at the probe  in allocate_trash_buffer()
447 		ret = ipu6_mmu_map(mmu->dmap->mmu_info, PFN_PHYS(iova_addr),  in allocate_trash_buffer()
448 				   mmu->pci_trash_page, PAGE_SIZE);  in allocate_trash_buffer()
450 			dev_err(mmu->dev,  in allocate_trash_buffer()
458 	mmu->iova_trash_page = PFN_PHYS(iova->pfn_lo);  in allocate_trash_buffer()
459 	dev_dbg(mmu->dev, "iova trash buffer for MMUID: %d is %u\n",  in allocate_trash_buffer()
460 		mmu->mmid, (unsigned int)mmu->iova_trash_page);  in allocate_trash_buffer()
464 	ipu6_mmu_unmap(mmu->dmap->mmu_info, PFN_PHYS(iova->pfn_lo),  in allocate_trash_buffer()
466 	dma_unmap_page(mmu->dmap->mmu_info->dev, mmu->pci_trash_page,  in allocate_trash_buffer()
469 	__free_iova(&mmu->dmap->iovad, iova);  in allocate_trash_buffer()
473 int ipu6_mmu_hw_init(struct ipu6_mmu *mmu)  in ipu6_mmu_hw_init()  argument
479 	mmu_info = mmu->dmap->mmu_info;  in ipu6_mmu_hw_init()
481 	/* Initialise the each MMU HW block */  in ipu6_mmu_hw_init()
482 	for (i = 0; i < mmu->nr_mmus; i++) {  in ipu6_mmu_hw_init()
483 		struct ipu6_mmu_hw *mmu_hw = &mmu->mmu_hw[i];  in ipu6_mmu_hw_init()
487 		/* Write page table address per MMU */  in ipu6_mmu_hw_init()
489 		       mmu->mmu_hw[i].base + REG_L1_PHYS);  in ipu6_mmu_hw_init()
491 		/* Set info bits per MMU */  in ipu6_mmu_hw_init()
492 		writel(mmu->mmu_hw[i].info_bits,  in ipu6_mmu_hw_init()
493 		       mmu->mmu_hw[i].base + REG_INFO);  in ipu6_mmu_hw_init()
495 		/* Configure MMU TLB stream configuration for L1 */  in ipu6_mmu_hw_init()
497 		     block_addr += mmu->mmu_hw[i].l1_block_sz[j], j++) {  in ipu6_mmu_hw_init()
499 				dev_err(mmu->dev, "invalid L1 configuration\n");  in ipu6_mmu_hw_init()
508 		/* Configure MMU TLB stream configuration for L2 */  in ipu6_mmu_hw_init()
510 		     block_addr += mmu->mmu_hw[i].l2_block_sz[j], j++) {  in ipu6_mmu_hw_init()
512 				dev_err(mmu->dev, "invalid L2 configuration\n");  in ipu6_mmu_hw_init()
521 	if (!mmu->trash_page) {  in ipu6_mmu_hw_init()
524 		mmu->trash_page = alloc_page(GFP_KERNEL);  in ipu6_mmu_hw_init()
525 		if (!mmu->trash_page) {  in ipu6_mmu_hw_init()
526 			dev_err(mmu->dev, "insufficient memory for trash buffer\n");  in ipu6_mmu_hw_init()
530 		ret = allocate_trash_buffer(mmu);  in ipu6_mmu_hw_init()
532 			__free_page(mmu->trash_page);  in ipu6_mmu_hw_init()
533 			mmu->trash_page = NULL;  in ipu6_mmu_hw_init()
534 			dev_err(mmu->dev, "trash buffer allocation failed\n");  in ipu6_mmu_hw_init()
539 	spin_lock_irqsave(&mmu->ready_lock, flags);  in ipu6_mmu_hw_init()
540 	mmu->ready = true;  in ipu6_mmu_hw_init()
541 	spin_unlock_irqrestore(&mmu->ready_lock, flags);  in ipu6_mmu_hw_init()
602 void ipu6_mmu_hw_cleanup(struct ipu6_mmu *mmu)  in ipu6_mmu_hw_cleanup()  argument
606 	spin_lock_irqsave(&mmu->ready_lock, flags);  in ipu6_mmu_hw_cleanup()
607 	mmu->ready = false;  in ipu6_mmu_hw_cleanup()
608 	spin_unlock_irqrestore(&mmu->ready_lock, flags);  in ipu6_mmu_hw_cleanup()
705 static void ipu6_mmu_destroy(struct ipu6_mmu *mmu)  in ipu6_mmu_destroy()  argument
707 	struct ipu6_dma_mapping *dmap = mmu->dmap;  in ipu6_mmu_destroy()
712 	if (mmu->iova_trash_page) {  in ipu6_mmu_destroy()
713 		iova = find_iova(&dmap->iovad, PHYS_PFN(mmu->iova_trash_page));  in ipu6_mmu_destroy()
720 			dev_err(mmu->dev, "trash buffer iova not found.\n");  in ipu6_mmu_destroy()
723 		mmu->iova_trash_page = 0;  in ipu6_mmu_destroy()
724 		dma_unmap_page(mmu_info->dev, mmu->pci_trash_page,  in ipu6_mmu_destroy()
726 		mmu->pci_trash_page = 0;  in ipu6_mmu_destroy()
727 		__free_page(mmu->trash_page);  in ipu6_mmu_destroy()
754 	struct ipu6_mmu *mmu;  in ipu6_mmu_init()  local
776 	mmu = devm_kzalloc(dev, sizeof(*mmu), GFP_KERNEL);  in ipu6_mmu_init()
777 	if (!mmu)  in ipu6_mmu_init()
780 	mmu->mmid = mmid;  in ipu6_mmu_init()
781 	mmu->mmu_hw = pdata->mmu_hw;  in ipu6_mmu_init()
782 	mmu->nr_mmus = hw->nr_mmus;  in ipu6_mmu_init()
783 	mmu->tlb_invalidate = tlb_invalidate;  in ipu6_mmu_init()
784 	mmu->ready = false;  in ipu6_mmu_init()
785 	INIT_LIST_HEAD(&mmu->vma_list);  in ipu6_mmu_init()
786 	spin_lock_init(&mmu->ready_lock);  in ipu6_mmu_init()
788 	mmu->dmap = alloc_dma_mapping(isp);  in ipu6_mmu_init()
789 	if (!mmu->dmap) {  in ipu6_mmu_init()
794 	return mmu;  in ipu6_mmu_init()
797 void ipu6_mmu_cleanup(struct ipu6_mmu *mmu)  in ipu6_mmu_cleanup()  argument
799 	struct ipu6_dma_mapping *dmap = mmu->dmap;  in ipu6_mmu_cleanup()
801 	ipu6_mmu_destroy(mmu);  in ipu6_mmu_cleanup()
802 	mmu->dmap = NULL;  in ipu6_mmu_cleanup()