Lines Matching +full:dma +full:- +full:33 +full:bits
1 // SPDX-License-Identifier: GPL-2.0-only
3 * CPU-agnostic ARM page table allocator.
5 * ARMv7 Short-descriptor format, supporting
6 * - Basic memory attributes
7 * - Simplified access permissions (AP[2:1] model)
8 * - Backwards-compatible TEX remap
9 * - Large pages/supersections (if indicated by the caller)
12 * - Legacy access permissions (AP[2:0] model)
15 * - PXN
16 * - Domains
18 * Copyright (C) 2014-2015 ARM Limited
19 * Copyright (c) 2014-2015 MediaTek Inc.
22 #define pr_fmt(fmt) "arm-v7s io-pgtable: " fmt
25 #include <linux/dma-mapping.h>
27 #include <linux/io-pgtable.h>
46 * We have 32 bits total; 12 bits resolved at level 1, 8 bits at level 2,
47 * and 12 bits in a page.
48 * MediaTek extend 2 bits to reach 34bits, 14 bits at lvl1 and 8 bits at lvl2.
51 #define _ARM_V7S_LVL_BITS(lvl, cfg) ((lvl) == 1 ? ((cfg)->ias - 20) : 8)
62 #define _ARM_V7S_IDX_MASK(lvl, cfg) (ARM_V7S_PTES_PER_LVL(lvl, cfg) - 1)
78 /* PTE type bits: these are all mixed up with XN/PXN bits in most cases */
87 /* Page table bits */
88 #define ARM_V7S_ATTR_XN(lvl) BIT(4 * (2 - (lvl)))
98 * The attribute bits are consistently ordered*, but occupy bits [17:10] of
99 * a level 1 PTE vs. bits [11:4] at level 2. Thus we define the individual
100 * fields relative to that 8-bit block, plus a total shift relative to the PTE.
102 #define ARM_V7S_ATTR_SHIFT(lvl) (16 - (lvl) * 6)
114 /* MediaTek extend the bits below for PA 32bit/33bit/34bit */
128 /* Register bits */
181 (cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_EXT); in arm_v7s_is_mtk_enabled()
188 if (paddr & BIT_ULL(33)) in to_mtk_iopte()
226 paddr |= BIT_ULL(33); in iopte_to_paddr()
235 return phys_to_virt(iopte_to_paddr(pte, lvl, &data->iop.cfg)); in iopte_deref()
241 struct io_pgtable_cfg *cfg = &data->iop.cfg; in __arm_v7s_alloc_table()
242 struct device *dev = cfg->iommu_dev; in __arm_v7s_alloc_table()
244 dma_addr_t dma; in __arm_v7s_alloc_table() local
253 gfp_l1 = cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT ? in __arm_v7s_alloc_table()
259 table = kmem_cache_zalloc(data->l2_tables, gfp); in __arm_v7s_alloc_table()
265 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT ? in __arm_v7s_alloc_table()
266 phys >= (1ULL << cfg->oas) : phys != (arm_v7s_iopte)phys) { in __arm_v7s_alloc_table()
271 if (!cfg->coherent_walk) { in __arm_v7s_alloc_table()
272 dma = dma_map_single(dev, table, size, DMA_TO_DEVICE); in __arm_v7s_alloc_table()
273 if (dma_mapping_error(dev, dma)) in __arm_v7s_alloc_table()
277 * address directly, so if the DMA layer suggests otherwise by in __arm_v7s_alloc_table()
280 if (dma != phys) in __arm_v7s_alloc_table()
288 dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n"); in __arm_v7s_alloc_table()
289 dma_unmap_single(dev, dma, size, DMA_TO_DEVICE); in __arm_v7s_alloc_table()
294 kmem_cache_free(data->l2_tables, table); in __arm_v7s_alloc_table()
301 struct io_pgtable_cfg *cfg = &data->iop.cfg; in __arm_v7s_free_table()
302 struct device *dev = cfg->iommu_dev; in __arm_v7s_free_table()
305 if (!cfg->coherent_walk) in __arm_v7s_free_table()
311 kmem_cache_free(data->l2_tables, table); in __arm_v7s_free_table()
317 if (cfg->coherent_walk) in __arm_v7s_pte_sync()
320 dma_sync_single_for_device(cfg->iommu_dev, __arm_v7s_dma_addr(ptep), in __arm_v7s_pte_sync()
337 bool ap = !(cfg->quirks & IO_PGTABLE_QUIRK_NO_PERMS); in arm_v7s_prot_to_pte()
359 if (lvl == 1 && (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)) in arm_v7s_prot_to_pte()
398 struct io_pgtable_cfg *cfg = &data->iop.cfg; in arm_v7s_init_pte()
411 tblp = ptep - ARM_V7S_LVL_IDX(iova, lvl, cfg); in arm_v7s_init_pte()
414 return -EINVAL; in arm_v7s_init_pte()
418 return -EEXIST; in arm_v7s_init_pte()
441 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT) in arm_v7s_install_table()
444 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) in arm_v7s_install_table()
464 struct io_pgtable_cfg *cfg = &data->iop.cfg; in __arm_v7s_map()
478 return -EINVAL; in __arm_v7s_map()
485 return -ENOMEM; in __arm_v7s_map()
500 return -EEXIST; in __arm_v7s_map()
512 int ret = -EINVAL; in arm_v7s_map_pages()
514 if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias) || in arm_v7s_map_pages()
515 paddr >= (1ULL << data->iop.cfg.oas))) in arm_v7s_map_pages()
516 return -ERANGE; in arm_v7s_map_pages()
519 return -EINVAL; in arm_v7s_map_pages()
521 while (pgcount--) { in arm_v7s_map_pages()
522 ret = __arm_v7s_map(data, iova, paddr, pgsize, prot, 1, data->pgd, in arm_v7s_map_pages()
545 for (i = 0; i < ARM_V7S_PTES_PER_LVL(1, &data->iop.cfg); i++) { in arm_v7s_free_pgtable()
546 arm_v7s_iopte pte = data->pgd[i]; in arm_v7s_free_pgtable()
552 __arm_v7s_free_table(data->pgd, 1, data); in arm_v7s_free_pgtable()
553 kmem_cache_destroy(data->l2_tables); in arm_v7s_free_pgtable()
563 struct io_pgtable *iop = &data->iop; in __arm_v7s_unmap()
570 idx = ARM_V7S_LVL_IDX(iova, lvl, &iop->cfg); in __arm_v7s_unmap()
584 * mark live entries being split. In practice (i.e. DMA API code), we in __arm_v7s_unmap()
597 __arm_v7s_set_pte(ptep, 0, num_entries, &iop->cfg); in __arm_v7s_unmap()
629 if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias))) in arm_v7s_unmap_pages()
632 while (pgcount--) { in arm_v7s_unmap_pages()
633 ret = __arm_v7s_unmap(data, gather, iova, pgsize, 1, data->pgd); in arm_v7s_unmap_pages()
648 arm_v7s_iopte *ptep = data->pgd, pte; in arm_v7s_iova_to_phys()
653 ptep += ARM_V7S_LVL_IDX(iova, ++lvl, &data->iop.cfg); in arm_v7s_iova_to_phys()
664 return iopte_to_paddr(pte, lvl, &data->iop.cfg) | (iova & ~mask); in arm_v7s_iova_to_phys()
674 if (cfg->ias > (arm_v7s_is_mtk_enabled(cfg) ? 34 : ARM_V7S_ADDR_BITS)) in arm_v7s_alloc_pgtable()
677 if (cfg->oas > (arm_v7s_is_mtk_enabled(cfg) ? 35 : ARM_V7S_ADDR_BITS)) in arm_v7s_alloc_pgtable()
680 if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | in arm_v7s_alloc_pgtable()
687 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_EXT && in arm_v7s_alloc_pgtable()
688 !(cfg->quirks & IO_PGTABLE_QUIRK_NO_PERMS)) in arm_v7s_alloc_pgtable()
691 if ((cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT) && in arm_v7s_alloc_pgtable()
703 slab_flag = cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT ? in arm_v7s_alloc_pgtable()
706 data->l2_tables = kmem_cache_create("io-pgtable_armv7s_l2", in arm_v7s_alloc_pgtable()
710 if (!data->l2_tables) in arm_v7s_alloc_pgtable()
713 data->iop.ops = (struct io_pgtable_ops) { in arm_v7s_alloc_pgtable()
720 data->iop.cfg = *cfg; in arm_v7s_alloc_pgtable()
726 cfg->pgsize_bitmap &= SZ_4K | SZ_64K | SZ_1M | SZ_16M; in arm_v7s_alloc_pgtable()
729 cfg->arm_v7s_cfg.tcr = 0; in arm_v7s_alloc_pgtable()
733 * under the non-TEX-remap interpretation of those attribute bits, in arm_v7s_alloc_pgtable()
734 * excepting various implementation-defined aspects of shareability. in arm_v7s_alloc_pgtable()
736 cfg->arm_v7s_cfg.prrr = ARM_V7S_PRRR_TR(1, ARM_V7S_PRRR_TYPE_DEVICE) | in arm_v7s_alloc_pgtable()
741 cfg->arm_v7s_cfg.nmrr = ARM_V7S_NMRR_IR(7, ARM_V7S_RGN_WBWA) | in arm_v7s_alloc_pgtable()
745 data->pgd = __arm_v7s_alloc_table(1, GFP_KERNEL, data); in arm_v7s_alloc_pgtable()
746 if (!data->pgd) in arm_v7s_alloc_pgtable()
753 paddr = virt_to_phys(data->pgd); in arm_v7s_alloc_pgtable()
755 cfg->arm_v7s_cfg.ttbr = paddr | upper_32_bits(paddr); in arm_v7s_alloc_pgtable()
757 cfg->arm_v7s_cfg.ttbr = paddr | ARM_V7S_TTBR_S | in arm_v7s_alloc_pgtable()
758 (cfg->coherent_walk ? (ARM_V7S_TTBR_NOS | in arm_v7s_alloc_pgtable()
763 return &data->iop; in arm_v7s_alloc_pgtable()
766 kmem_cache_destroy(data->l2_tables); in arm_v7s_alloc_pgtable()
789 WARN_ON(!(size & cfg_cookie->pgsize_bitmap)); in dummy_tlb_flush()
808 -EFAULT; \
833 return -EINVAL; in arm_v7s_do_selftests()
840 if (ops->iova_to_phys(ops, 42)) in arm_v7s_do_selftests()
843 if (ops->iova_to_phys(ops, SZ_1G + 42)) in arm_v7s_do_selftests()
846 if (ops->iova_to_phys(ops, SZ_2G + 42)) in arm_v7s_do_selftests()
855 if (ops->map_pages(ops, iova, iova, size, 1, in arm_v7s_do_selftests()
862 if (!ops->map_pages(ops, iova, iova + size, size, 1, in arm_v7s_do_selftests()
867 if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) in arm_v7s_do_selftests()
878 if (ops->unmap_pages(ops, iova, size, 1, NULL) != size) in arm_v7s_do_selftests()
881 if (ops->iova_to_phys(ops, iova + 42)) in arm_v7s_do_selftests()
885 if (ops->map_pages(ops, iova, iova, size, 1, IOMMU_WRITE, in arm_v7s_do_selftests()
889 if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) in arm_v7s_do_selftests()