32 	container_of((x), struct arm_lpae_io_pgtable, iop)
166 	struct io_pgtable	iop;  member
348 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in __arm_lpae_init_pte()
352 	if (data->iop.fmt != ARM_MALI_LPAE && lvl == ARM_LPAE_MAX_LEVELS - 1)  in __arm_lpae_init_pte()
372 		if (iopte_leaf(ptep[i], lvl, data->iop.fmt)) {  in arm_lpae_init_pte()
402 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_install_table()
436 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in __arm_lpae_map()
461 		cptep = __arm_lpae_alloc_pages(tblsz, gfp, cfg, data->iop.cookie);  in __arm_lpae_map()
467 			__arm_lpae_free_pages(cptep, tblsz, cfg, data->iop.cookie);  in __arm_lpae_map()
472 	if (pte && !iopte_leaf(pte, lvl, data->iop.fmt)) {  in __arm_lpae_map()
490 	if (data->iop.fmt == ARM_64_LPAE_S1 ||  in arm_lpae_prot_to_pte()
491 	    data->iop.fmt == ARM_32_LPAE_S1) {  in arm_lpae_prot_to_pte()
495 		else if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_HD)  in arm_lpae_prot_to_pte()
511 	if (data->iop.fmt == ARM_64_LPAE_S2 ||  in arm_lpae_prot_to_pte()
512 	    data->iop.fmt == ARM_32_LPAE_S2) {  in arm_lpae_prot_to_pte()
516 			if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_S2FWB)  in arm_lpae_prot_to_pte()
538 	if (prot & IOMMU_CACHE && data->iop.fmt != ARM_MALI_LPAE)  in arm_lpae_prot_to_pte()
546 	if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS)  in arm_lpae_prot_to_pte()
549 	if (data->iop.fmt != ARM_MALI_LPAE)  in arm_lpae_prot_to_pte()
560 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_map_pages()
611 		if (!pte || iopte_leaf(pte, lvl, data->iop.fmt))  in __arm_lpae_free_pgtable()
617 	__arm_lpae_free_pages(start, table_size, &data->iop.cfg, data->iop.cookie);  in __arm_lpae_free_pgtable()
620 static void arm_lpae_free_pgtable(struct io_pgtable *iop)  in arm_lpae_free_pgtable()  argument
622 	struct arm_lpae_io_pgtable *data = io_pgtable_to_data(iop);  in arm_lpae_free_pgtable()
634 	struct io_pgtable *iop = &data->iop;  in __arm_lpae_unmap()  local
658 			if (!iopte_leaf(pte, lvl, iop->fmt)) {  in __arm_lpae_unmap()
659 				__arm_lpae_clear_pte(&ptep[i], &iop->cfg, 1);  in __arm_lpae_unmap()
662 				io_pgtable_tlb_flush_walk(iop, iova + i * size, size,  in __arm_lpae_unmap()
669 		__arm_lpae_clear_pte(ptep, &iop->cfg, i);  in __arm_lpae_unmap()
673 				io_pgtable_tlb_add_page(iop, gather, iova + j * size, size);  in __arm_lpae_unmap()
676 	} else if (iopte_leaf(pte, lvl, iop->fmt)) {  in __arm_lpae_unmap()
691 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_unmap_pages()
708 	struct io_pgtable		*iop;  member
783 	struct io_pgtable *iop = &data->iop;  in io_pgtable_visit()  local
791 	if (iopte_leaf(pte, lvl, iop->fmt)) {  in io_pgtable_visit()
835 	if (!iopte_leaf(*ptep, lvl, walk_data->iop->fmt))  in visit_dirty()
853 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_read_and_clear_dirty()
855 		.iop = &data->iop,  in arm_lpae_read_and_clear_dirty()
869 	if (data->iop.fmt != ARM_64_LPAE_S1)  in arm_lpae_read_and_clear_dirty()
949 	data->iop.ops = (struct io_pgtable_ops) {  in arm_lpae_alloc_pgtable()
1058 	return &data->iop;  in arm_64_lpae_alloc_pgtable_s1()
1153 	return &data->iop;  in arm_64_lpae_alloc_pgtable_s2()
1232 	return &data->iop;  in arm_mali_lpae_alloc_pgtable()
1301 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_dump_ops()