Lines Matching +full:disable +full:- +full:mmu +full:- +full:reset
1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2022 HabanaLabs, Ltd.
16 * hl_mmu_get_funcs() - get MMU functions structure
21 * @return appropriate MMU functions structure
26 return &hdev->mmu_func[pgt_residency]; in hl_mmu_get_funcs()
31 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_is_dram_va()
33 return hl_mem_area_inside_range(virt_addr, prop->dmmu.page_size, in hl_is_dram_va()
34 prop->dmmu.start_addr, in hl_is_dram_va()
35 prop->dmmu.end_addr); in hl_is_dram_va()
39 * hl_mmu_init() - initialize the MMU module.
42 * Return: 0 for success, non-zero for failure.
46 int rc = -EOPNOTSUPP; in hl_mmu_init()
48 if (hdev->mmu_disable) in hl_mmu_init()
51 mutex_init(&hdev->mmu_lock); in hl_mmu_init()
53 if (hdev->mmu_func[MMU_DR_PGT].init != NULL) { in hl_mmu_init()
54 rc = hdev->mmu_func[MMU_DR_PGT].init(hdev); in hl_mmu_init()
59 if (hdev->mmu_func[MMU_HR_PGT].init != NULL) { in hl_mmu_init()
60 rc = hdev->mmu_func[MMU_HR_PGT].init(hdev); in hl_mmu_init()
68 if (hdev->mmu_func[MMU_DR_PGT].fini != NULL) in hl_mmu_init()
69 hdev->mmu_func[MMU_DR_PGT].fini(hdev); in hl_mmu_init()
75 * hl_mmu_fini() - release the MMU module.
79 * - Disable MMU in H/W.
80 * - Free the pgt_infos pool.
86 if (hdev->mmu_disable) in hl_mmu_fini()
89 if (hdev->mmu_func[MMU_DR_PGT].fini != NULL) in hl_mmu_fini()
90 hdev->mmu_func[MMU_DR_PGT].fini(hdev); in hl_mmu_fini()
92 if (hdev->mmu_func[MMU_HR_PGT].fini != NULL) in hl_mmu_fini()
93 hdev->mmu_func[MMU_HR_PGT].fini(hdev); in hl_mmu_fini()
95 mutex_destroy(&hdev->mmu_lock); in hl_mmu_fini()
99 * hl_mmu_ctx_init() - initialize a context for using the MMU module.
104 * Return: 0 on success, non-zero otherwise.
108 struct hl_device *hdev = ctx->hdev; in hl_mmu_ctx_init()
109 int rc = -EOPNOTSUPP; in hl_mmu_ctx_init()
111 if (hdev->mmu_disable) in hl_mmu_ctx_init()
114 if (hdev->mmu_func[MMU_DR_PGT].ctx_init != NULL) { in hl_mmu_ctx_init()
115 rc = hdev->mmu_func[MMU_DR_PGT].ctx_init(ctx); in hl_mmu_ctx_init()
120 if (hdev->mmu_func[MMU_HR_PGT].ctx_init != NULL) { in hl_mmu_ctx_init()
121 rc = hdev->mmu_func[MMU_HR_PGT].ctx_init(ctx); in hl_mmu_ctx_init()
129 if (hdev->mmu_func[MMU_DR_PGT].fini != NULL) in hl_mmu_ctx_init()
130 hdev->mmu_func[MMU_DR_PGT].fini(hdev); in hl_mmu_ctx_init()
136 * hl_mmu_ctx_fini - disable a ctx from using the mmu module
141 * - Free any pgts which were not freed yet
142 * - Free the mutex
143 * - Free DRAM default page mapping hops
147 struct hl_device *hdev = ctx->hdev; in hl_mmu_ctx_fini()
149 if (hdev->mmu_disable) in hl_mmu_ctx_fini()
152 if (hdev->mmu_func[MMU_DR_PGT].ctx_fini != NULL) in hl_mmu_ctx_fini()
153 hdev->mmu_func[MMU_DR_PGT].ctx_fini(ctx); in hl_mmu_ctx_fini()
155 if (hdev->mmu_func[MMU_HR_PGT].ctx_fini != NULL) in hl_mmu_ctx_fini()
156 hdev->mmu_func[MMU_HR_PGT].ctx_fini(ctx); in hl_mmu_ctx_fini()
160 * hl_mmu_get_real_page_size - get real page size to use in map/unmap operation
163 * @mmu_prop: MMU properties.
170 * note that this is general implementation that can fit most MMU arch. but as this is used as an
171 * MMU function:
172 * 1. it shall not be called directly- only from mmu_func structure instance
173 * 2. each MMU may modify the implementation internally
180 * size is bigger, we break it to sub-pages and map them separately. in hl_mmu_get_real_page_size()
182 if ((page_size % mmu_prop->page_size) == 0) { in hl_mmu_get_real_page_size()
183 *real_page_size = mmu_prop->page_size; in hl_mmu_get_real_page_size()
187 dev_err(hdev->dev, "page size of %u is not %uKB aligned, can't map\n", in hl_mmu_get_real_page_size()
188 page_size, mmu_prop->page_size >> 10); in hl_mmu_get_real_page_size()
190 return -EFAULT; in hl_mmu_get_real_page_size()
196 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_mmu_get_prop()
199 return &prop->dmmu; in hl_mmu_get_prop()
200 else if ((page_size % prop->pmmu_huge.page_size) == 0) in hl_mmu_get_prop()
201 return &prop->pmmu_huge; in hl_mmu_get_prop()
203 return &prop->pmmu; in hl_mmu_get_prop()
207 * hl_mmu_unmap_page - unmaps a virtual addr
215 * - Check that the virt addr is mapped
216 * - Unmap the virt addr and frees pgts if possible
217 * - Returns 0 on success, -EINVAL if the given addr is not mapped
220 * changes the MMU hash, it must be protected by a lock.
229 struct hl_device *hdev = ctx->hdev; in hl_mmu_unmap_page()
237 if (hdev->mmu_disable) in hl_mmu_unmap_page()
243 pgt_residency = mmu_prop->host_resident ? MMU_HR_PGT : MMU_DR_PGT; in hl_mmu_unmap_page()
246 rc = hdev->asic_funcs->mmu_get_real_page_size(hdev, mmu_prop, page_size, &real_page_size, in hl_mmu_unmap_page()
255 rc = mmu_funcs->unmap(ctx, real_virt_addr, is_dram_addr); in hl_mmu_unmap_page()
263 mmu_funcs->flush(ctx); in hl_mmu_unmap_page()
266 trace_habanalabs_mmu_unmap(&hdev->pdev->dev, virt_addr, 0, page_size, flush_pte); in hl_mmu_unmap_page()
272 * hl_mmu_map_page - maps a virtual addr to physical addr
281 * - Check that the virt addr is not mapped
282 * - Allocate pgts as necessary in order to map the virt addr to the phys
283 * - Returns 0 on success, -EINVAL if addr is already mapped, or -ENOMEM.
286 * changes the MMU hash, it must be protected by a lock.
297 struct hl_device *hdev = ctx->hdev; in hl_mmu_map_page()
305 if (hdev->mmu_disable) in hl_mmu_map_page()
311 pgt_residency = mmu_prop->host_resident ? MMU_HR_PGT : MMU_DR_PGT; in hl_mmu_map_page()
314 rc = hdev->asic_funcs->mmu_get_real_page_size(hdev, mmu_prop, page_size, &real_page_size, in hl_mmu_map_page()
321 * MMU page size (in dram this means checking the address and MMU in hl_mmu_map_page()
325 ((hdev->asic_funcs->scramble_addr(hdev, phys_addr) & in hl_mmu_map_page()
326 (mmu_prop->page_size - 1)) || in hl_mmu_map_page()
327 (hdev->asic_funcs->scramble_addr(hdev, virt_addr) & in hl_mmu_map_page()
328 (mmu_prop->page_size - 1)))) || in hl_mmu_map_page()
329 (!is_dram_addr && ((phys_addr & (real_page_size - 1)) || in hl_mmu_map_page()
330 (virt_addr & (real_page_size - 1))))) in hl_mmu_map_page()
331 dev_crit(hdev->dev, in hl_mmu_map_page()
340 rc = mmu_funcs->map(ctx, real_virt_addr, real_phys_addr, real_page_size, in hl_mmu_map_page()
351 mmu_funcs->flush(ctx); in hl_mmu_map_page()
353 trace_habanalabs_mmu_map(&hdev->pdev->dev, virt_addr, phys_addr, page_size, flush_pte); in hl_mmu_map_page()
360 if (mmu_funcs->unmap(ctx, real_virt_addr, is_dram_addr)) in hl_mmu_map_page()
361 dev_warn_ratelimited(hdev->dev, in hl_mmu_map_page()
367 mmu_funcs->flush(ctx); in hl_mmu_map_page()
373 * hl_mmu_map_contiguous - implements a wrapper for hl_mmu_map_page
385 struct hl_device *hdev = ctx->hdev; in hl_mmu_map_contiguous()
386 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_mmu_map_contiguous()
393 prop->dmmu.start_addr, prop->dmmu.end_addr)) in hl_mmu_map_contiguous()
394 page_size = prop->dmmu.page_size; in hl_mmu_map_contiguous()
396 prop->pmmu.start_addr, prop->pmmu.end_addr)) in hl_mmu_map_contiguous()
397 page_size = prop->pmmu.page_size; in hl_mmu_map_contiguous()
399 prop->pmmu_huge.start_addr, prop->pmmu_huge.end_addr)) in hl_mmu_map_contiguous()
400 page_size = prop->pmmu_huge.page_size; in hl_mmu_map_contiguous()
402 return -EINVAL; in hl_mmu_map_contiguous()
411 dev_err(hdev->dev, in hl_mmu_map_contiguous()
414 /* last mapping failed so don't try to unmap it - reduce off by page_size */ in hl_mmu_map_contiguous()
415 off -= page_size; in hl_mmu_map_contiguous()
423 for (; off >= 0 ; off -= page_size) { in hl_mmu_map_contiguous()
425 flush_pte = (off - (s32) page_size) < 0; in hl_mmu_map_contiguous()
427 dev_warn_ratelimited(hdev->dev, in hl_mmu_map_contiguous()
435 * hl_mmu_unmap_contiguous - implements a wrapper for hl_mmu_unmap_page
445 struct hl_device *hdev = ctx->hdev; in hl_mmu_unmap_contiguous()
446 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_mmu_unmap_contiguous()
453 prop->dmmu.start_addr, prop->dmmu.end_addr)) in hl_mmu_unmap_contiguous()
454 page_size = prop->dmmu.page_size; in hl_mmu_unmap_contiguous()
456 prop->pmmu.start_addr, prop->pmmu.end_addr)) in hl_mmu_unmap_contiguous()
457 page_size = prop->pmmu.page_size; in hl_mmu_unmap_contiguous()
459 prop->pmmu_huge.start_addr, prop->pmmu_huge.end_addr)) in hl_mmu_unmap_contiguous()
460 page_size = prop->pmmu_huge.page_size; in hl_mmu_unmap_contiguous()
462 return -EINVAL; in hl_mmu_unmap_contiguous()
469 dev_warn_ratelimited(hdev->dev, in hl_mmu_unmap_contiguous()
480 struct asic_fixed_properties *prop = &ctx->hdev->asic_prop; in hl_mmu_pa_page_with_offset()
485 if (hops->unscrambled_paddr) in hl_mmu_pa_page_with_offset()
486 tmp_phys_addr = hops->unscrambled_paddr; in hl_mmu_pa_page_with_offset()
488 tmp_phys_addr = hops->hop_info[hops->used_hops - 1].hop_pte_val; in hl_mmu_pa_page_with_offset()
490 if (hops->range_type == HL_VA_RANGE_TYPE_HOST_HUGE) in hl_mmu_pa_page_with_offset()
491 mmu_prop = &prop->pmmu_huge; in hl_mmu_pa_page_with_offset()
492 else if (hops->range_type == HL_VA_RANGE_TYPE_HOST) in hl_mmu_pa_page_with_offset()
493 mmu_prop = &prop->pmmu; in hl_mmu_pa_page_with_offset()
495 mmu_prop = &prop->dmmu; in hl_mmu_pa_page_with_offset()
497 if ((hops->range_type == HL_VA_RANGE_TYPE_DRAM) && in hl_mmu_pa_page_with_offset()
498 !is_power_of_2(prop->dram_page_size)) { in hl_mmu_pa_page_with_offset()
509 dram_page_size = prop->dram_page_size; in hl_mmu_pa_page_with_offset()
510 dram_base = prop->dram_base_address; in hl_mmu_pa_page_with_offset()
511 abs_phys_addr = tmp_phys_addr - dram_base; in hl_mmu_pa_page_with_offset()
512 abs_virt_addr = virt_addr - dram_base; in hl_mmu_pa_page_with_offset()
524 hop_shift = mmu_prop->hop_shifts[hops->used_hops - 1]; in hl_mmu_pa_page_with_offset()
525 offset_mask = (1ull << hop_shift) - 1; in hl_mmu_pa_page_with_offset()
551 struct hl_device *hdev = ctx->hdev; in hl_mmu_get_tlb_info()
558 if (hdev->mmu_disable) in hl_mmu_get_tlb_info()
559 return -EOPNOTSUPP; in hl_mmu_get_tlb_info()
561 prop = &hdev->asic_prop; in hl_mmu_get_tlb_info()
562 hops->scrambled_vaddr = virt_addr; /* assume no scrambling */ in hl_mmu_get_tlb_info()
564 is_dram_addr = hl_mem_area_inside_range(virt_addr, prop->dmmu.page_size, in hl_mmu_get_tlb_info()
565 prop->dmmu.start_addr, in hl_mmu_get_tlb_info()
566 prop->dmmu.end_addr); in hl_mmu_get_tlb_info()
568 /* host-residency is the same in PMMU and PMMU huge, no need to distinguish here */ in hl_mmu_get_tlb_info()
569 mmu_prop = is_dram_addr ? &prop->dmmu : &prop->pmmu; in hl_mmu_get_tlb_info()
570 pgt_residency = mmu_prop->host_resident ? MMU_HR_PGT : MMU_DR_PGT; in hl_mmu_get_tlb_info()
573 mutex_lock(&hdev->mmu_lock); in hl_mmu_get_tlb_info()
574 rc = mmu_funcs->get_tlb_info(ctx, virt_addr, hops); in hl_mmu_get_tlb_info()
575 mutex_unlock(&hdev->mmu_lock); in hl_mmu_get_tlb_info()
581 if (hops->unscrambled_paddr) in hl_mmu_get_tlb_info()
582 hl_mmu_pa_page_with_offset(ctx, virt_addr, hops, &hops->unscrambled_paddr); in hl_mmu_get_tlb_info()
589 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_mmu_if_set_funcs()
591 if (hdev->mmu_disable) in hl_mmu_if_set_funcs()
594 switch (hdev->asic_type) { in hl_mmu_if_set_funcs()
598 hl_mmu_v1_set_funcs(hdev, &hdev->mmu_func[MMU_DR_PGT]); in hl_mmu_if_set_funcs()
604 hl_mmu_v2_set_funcs(hdev, &hdev->mmu_func[MMU_DR_PGT]); in hl_mmu_if_set_funcs()
605 if (prop->pmmu.host_resident) in hl_mmu_if_set_funcs()
606 hl_mmu_v2_hr_set_funcs(hdev, &hdev->mmu_func[MMU_HR_PGT]); in hl_mmu_if_set_funcs()
609 dev_err(hdev->dev, "Unrecognized ASIC type %d\n", in hl_mmu_if_set_funcs()
610 hdev->asic_type); in hl_mmu_if_set_funcs()
611 return -EOPNOTSUPP; in hl_mmu_if_set_funcs()
618 * hl_mmu_scramble_addr() - The generic mmu address scrambling routine.
630 * hl_mmu_descramble_addr() - The generic mmu address descrambling
635 * Return: The un-scrambled address.
646 rc = hdev->asic_funcs->mmu_invalidate_cache(hdev, is_hard, flags); in hl_mmu_invalidate_cache()
648 dev_err_ratelimited(hdev->dev, in hl_mmu_invalidate_cache()
650 dev_name(&hdev->pdev->dev), in hl_mmu_invalidate_cache()
661 rc = hdev->asic_funcs->mmu_invalidate_cache_range(hdev, is_hard, flags, in hl_mmu_invalidate_cache_range()
664 dev_err_ratelimited(hdev->dev, in hl_mmu_invalidate_cache_range()
666 dev_name(&hdev->pdev->dev), flags == VM_TYPE_USERPTR ? "PMMU" : "HMMU", in hl_mmu_invalidate_cache_range()
675 struct hl_ctx *ctx = pfw->ctx; in hl_mmu_prefetch_work_function()
676 struct hl_device *hdev = ctx->hdev; in hl_mmu_prefetch_work_function()
681 mutex_lock(&hdev->mmu_lock); in hl_mmu_prefetch_work_function()
683 hdev->asic_funcs->mmu_prefetch_cache_range(ctx, pfw->flags, pfw->asid, pfw->va, pfw->size); in hl_mmu_prefetch_work_function()
685 mutex_unlock(&hdev->mmu_lock); in hl_mmu_prefetch_work_function()
689 * context was taken in the common mmu prefetch function- see comment there about in hl_mmu_prefetch_work_function()
702 return -ENOMEM; in hl_mmu_prefetch_cache_range()
704 INIT_WORK(&handle_prefetch_work->prefetch_work, hl_mmu_prefetch_work_function); in hl_mmu_prefetch_cache_range()
705 handle_prefetch_work->ctx = ctx; in hl_mmu_prefetch_cache_range()
706 handle_prefetch_work->va = va; in hl_mmu_prefetch_cache_range()
707 handle_prefetch_work->size = size; in hl_mmu_prefetch_cache_range()
708 handle_prefetch_work->flags = flags; in hl_mmu_prefetch_cache_range()
709 handle_prefetch_work->asid = asid; in hl_mmu_prefetch_cache_range()
716 queue_work(ctx->hdev->prefetch_wq, &handle_prefetch_work->prefetch_work); in hl_mmu_prefetch_cache_range()
727 * hl_mmu_get_hop_pte_phys_addr() - extract PTE address from HOP
729 * @mmu_prop: MMU properties.
741 if (hop_idx >= mmu_prop->num_hops) { in hl_mmu_get_hop_pte_phys_addr()
742 dev_err_ratelimited(ctx->hdev->dev, "Invalid hop index %d\n", hop_idx); in hl_mmu_get_hop_pte_phys_addr()
746 shift = mmu_prop->hop_shifts[hop_idx]; in hl_mmu_get_hop_pte_phys_addr()
747 mask = mmu_prop->hop_masks[hop_idx]; in hl_mmu_get_hop_pte_phys_addr()
749 return hop_addr + ctx->hdev->asic_prop.mmu_pte_size * ((virt_addr & mask) >> shift); in hl_mmu_get_hop_pte_phys_addr()
758 hl_asic_dma_free_coherent(hdev, (chunk->end_addr - chunk->start_addr) + 1, in mmu_dma_mem_free_from_chunk()
759 (void *)chunk->start_addr, chunk->phys_addr); in mmu_dma_mem_free_from_chunk()
769 * hl_mmu_hr_pool_destroy() - destroy genpool
771 * @hr_priv: MMU HR private data.
775 * - free entries allocated for shadow HOP0
776 * - free pool chunks
777 * - free pool
782 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_mmu_hr_pool_destroy()
783 struct gen_pool **pool = &hr_priv->mmu_pgt_pool; in hl_mmu_hr_pool_destroy()
791 if (hr_priv->mmu_asid_hop0) { in hl_mmu_hr_pool_destroy()
792 for (asid = 0 ; asid < prop->max_asid ; asid++) { in hl_mmu_hr_pool_destroy()
793 hop0_pgt = &hr_priv->mmu_asid_hop0[asid]; in hl_mmu_hr_pool_destroy()
794 if (ZERO_OR_NULL_PTR(hop0_pgt->virt_addr)) in hl_mmu_hr_pool_destroy()
797 gen_pool_free(*pool, (uintptr_t) hop0_pgt->virt_addr, hop_table_size); in hl_mmu_hr_pool_destroy()
806 * during hard reset code at certain points in hl_mmu_hr_pool_destroy()
812 * hl_mmu_hr_init() - initialize the MMU module.
814 * @hr_priv: MMU HR private data.
818 * @return 0 on success otherwise non-zero error code
821 * - Create a pool of pages for pgt_infos.
822 * - Create a shadow table for pgt
827 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_mmu_hr_init()
838 * allocations with higher-order alignment restrictions in hl_mmu_hr_init()
840 hr_priv->mmu_pgt_pool = gen_pool_create(PAGE_SHIFT, -1); in hl_mmu_hr_init()
841 if (ZERO_OR_NULL_PTR(hr_priv->mmu_pgt_pool)) { in hl_mmu_hr_init()
842 dev_err(hdev->dev, "Failed to create hr page pool\n"); in hl_mmu_hr_init()
843 return -ENOMEM; in hl_mmu_hr_init()
846 hr_priv->mmu_asid_hop0 = kvcalloc(prop->max_asid, sizeof(struct pgt_info), GFP_KERNEL); in hl_mmu_hr_init()
847 if (ZERO_OR_NULL_PTR(hr_priv->mmu_asid_hop0)) { in hl_mmu_hr_init()
848 dev_err(hdev->dev, "Failed to allocate hr-mmu hop0 table\n"); in hl_mmu_hr_init()
849 rc = -ENOMEM; in hl_mmu_hr_init()
858 dev_err(hdev->dev, in hl_mmu_hr_init()
859 "Failed to allocate memory for host-resident page pool\n"); in hl_mmu_hr_init()
860 rc = -ENOMEM; in hl_mmu_hr_init()
864 rc = gen_pool_add_virt(hr_priv->mmu_pgt_pool, virt_addr, (phys_addr_t) dma_addr, in hl_mmu_hr_init()
865 pool_chunk_size, -1); in hl_mmu_hr_init()
867 dev_err(hdev->dev, "Failed to fill host-resident page pool\n"); in hl_mmu_hr_init()
872 for (i = 0 ; i < prop->max_asid ; i++) { in hl_mmu_hr_init()
873 hop0_pgt = &hr_priv->mmu_asid_hop0[i]; in hl_mmu_hr_init()
874 hop0_pgt->virt_addr = (uintptr_t) in hl_mmu_hr_init()
875 gen_pool_dma_zalloc_align(hr_priv->mmu_pgt_pool, in hl_mmu_hr_init()
877 (dma_addr_t *) &hop0_pgt->phys_addr, in hl_mmu_hr_init()
879 if (!hop0_pgt->virt_addr) { in hl_mmu_hr_init()
880 dev_err(hdev->dev, "Failed to allocate HOP from pgt pool\n"); in hl_mmu_hr_init()
881 rc = -ENOMEM; in hl_mmu_hr_init()
886 /* MMU H/W init will be done in device hw_init() */ in hl_mmu_hr_init()
892 if (!ZERO_OR_NULL_PTR(hr_priv->mmu_asid_hop0)) in hl_mmu_hr_init()
893 kvfree(hr_priv->mmu_asid_hop0); in hl_mmu_hr_init()
899 * hl_mmu_hr_fini() - release the MMU module.
901 * @hr_priv: MMU host resident private info.
905 * - Disable MMU in H/W.
906 * - Free the pgt_infos pool.
912 /* MMU H/W fini was already done in device hw_fini() */ in hl_mmu_hr_fini()
916 if (!ZERO_OR_NULL_PTR(hr_priv->mmu_asid_hop0)) { in hl_mmu_hr_fini()
917 kvfree(hr_priv->mmu_asid_hop0); in hl_mmu_hr_fini()
921 * example if we fail during hard reset code at certain points in hl_mmu_hr_fini()
923 hr_priv->mmu_asid_hop0 = NULL; in hl_mmu_hr_fini()
928 * hl_mmu_hr_free_hop_remove_pgt() - free HOP and remove PGT from hash
930 * @hr_priv: MMU HR private data.
936 gen_pool_free(hr_priv->mmu_pgt_pool, pgt_info->virt_addr, hop_table_size); in hl_mmu_hr_free_hop_remove_pgt()
937 hash_del(&pgt_info->node); in hl_mmu_hr_free_hop_remove_pgt()
942 * hl_mmu_hr_pte_phys_to_virt() - translate PTE phys addr to virt addr
956 u64 page_mask = (hop_table_size - 1); in hl_mmu_hr_pte_phys_to_virt()
959 return pgt->virt_addr + pte_offset; in hl_mmu_hr_pte_phys_to_virt()
963 * hl_mmu_hr_write_pte() - write HR PTE
983 * hl_mmu_hr_clear_pte() - clear HR PTE
997 * hl_mmu_hr_put_pte() - put HR PTE and remove it if necessary (no more PTEs)
1000 * @hr_priv: HR MMU private info
1011 pgt_info->num_of_ptes--; in hl_mmu_hr_put_pte()
1017 num_of_ptes_left = pgt_info->num_of_ptes; in hl_mmu_hr_put_pte()
1025 * hl_mmu_hr_get_pte() - increase PGT PTE count
1032 hr_func->get_pgt_info(ctx, phys_hop_addr)->num_of_ptes++; in hl_mmu_hr_get_pte()
1036 * hl_mmu_hr_get_next_hop_pgt_info() - get pgt_info structure for the next HOP
1052 return hr_func->get_pgt_info(ctx, next_hop_phys_addr); in hl_mmu_hr_get_next_hop_pgt_info()
1056 * hl_mmu_hr_alloc_hop() - allocate HOP
1060 * @mmu_prop: MMU properties.
1068 struct hl_device *hdev = ctx->hdev; in hl_mmu_hr_alloc_hop()
1079 virt_addr = gen_pool_dma_zalloc_align(hr_priv->mmu_pgt_pool, in hl_mmu_hr_alloc_hop()
1080 mmu_prop->hop_table_size, in hl_mmu_hr_alloc_hop()
1082 mmu_prop->hop_table_size); in hl_mmu_hr_alloc_hop()
1086 /* No memory in pool - get some and try again */ in hl_mmu_hr_alloc_hop()
1092 if (gen_pool_add_virt(hr_priv->mmu_pgt_pool, (unsigned long)virt_addr, in hl_mmu_hr_alloc_hop()
1093 phys_addr, SZ_2M, -1)) { in hl_mmu_hr_alloc_hop()
1101 dev_err(hdev->dev, "failed to allocate page\n"); in hl_mmu_hr_alloc_hop()
1105 pgt_info->phys_addr = phys_addr; in hl_mmu_hr_alloc_hop()
1106 pgt_info->shadow_addr = (unsigned long) NULL; in hl_mmu_hr_alloc_hop()
1107 pgt_info->virt_addr = (unsigned long)virt_addr; in hl_mmu_hr_alloc_hop()
1108 pgt_info->ctx = ctx; in hl_mmu_hr_alloc_hop()
1109 pgt_info->num_of_ptes = 0; in hl_mmu_hr_alloc_hop()
1110 hr_func->add_pgt_info(ctx, pgt_info, phys_addr); in hl_mmu_hr_alloc_hop()
1121 * hl_mmu_hr_get_alloc_next_hop() - get the next HOP, allocate it if it does not exist
1125 * @mmu_prop: MMU properties.
1140 return hr_func->get_pgt_info(ctx, hop_addr); in hl_mmu_hr_get_alloc_next_hop()
1147 * hl_mmu_hr_get_tlb_info() - get the TLB info (info for a specific mapping)
1160 struct hl_device *hdev = ctx->hdev; in hl_mmu_hr_get_tlb_info()
1165 rc = hr_func->get_tlb_mapping_params(hdev, &mmu_prop, hops, virt_addr, &is_huge); in hl_mmu_hr_get_tlb_info()
1169 used_hops = mmu_prop->num_hops; in hl_mmu_hr_get_tlb_info()
1173 used_hops--; in hl_mmu_hr_get_tlb_info()
1175 hops->scrambled_vaddr = hdev->asic_funcs->scramble_addr(hdev, virt_addr); in hl_mmu_hr_get_tlb_info()
1179 hops_pgt_info[i] = hr_func->get_hop0_pgt_info(ctx); in hl_mmu_hr_get_tlb_info()
1182 hops->hop_info[i - 1].hop_pte_val); in hl_mmu_hr_get_tlb_info()
1185 return -EFAULT; in hl_mmu_hr_get_tlb_info()
1187 hops->hop_info[i].hop_addr = hops_pgt_info[i]->phys_addr; in hl_mmu_hr_get_tlb_info()
1188 hops->hop_info[i].hop_pte_addr = in hl_mmu_hr_get_tlb_info()
1190 hops->hop_info[i].hop_addr, in hl_mmu_hr_get_tlb_info()
1191 hops->scrambled_vaddr); in hl_mmu_hr_get_tlb_info()
1192 hops->hop_info[i].hop_pte_val = *(u64 *) (uintptr_t) in hl_mmu_hr_get_tlb_info()
1194 hops->hop_info[i].hop_pte_addr, in hl_mmu_hr_get_tlb_info()
1195 mmu_prop->hop_table_size); in hl_mmu_hr_get_tlb_info()
1197 if (!(hops->hop_info[i].hop_pte_val & PAGE_PRESENT_MASK)) in hl_mmu_hr_get_tlb_info()
1198 return -EFAULT; in hl_mmu_hr_get_tlb_info()
1200 if (hops->hop_info[i].hop_pte_val & mmu_prop->last_mask) in hl_mmu_hr_get_tlb_info()
1205 if (i == mmu_prop->num_hops) in hl_mmu_hr_get_tlb_info()
1206 return -EFAULT; in hl_mmu_hr_get_tlb_info()
1208 if (hops->scrambled_vaddr != virt_addr) in hl_mmu_hr_get_tlb_info()
1209 hops->unscrambled_paddr = hdev->asic_funcs->descramble_addr in hl_mmu_hr_get_tlb_info()
1210 (hdev, hops->hop_info[i].hop_pte_val); in hl_mmu_hr_get_tlb_info()
1212 hops->unscrambled_paddr = hops->hop_info[i].hop_pte_val; in hl_mmu_hr_get_tlb_info()
1214 hops->used_hops = i + 1; in hl_mmu_hr_get_tlb_info()
1223 hash_for_each_possible(ctx->mmu_shadow_hash, pgt_info, node, in hl_mmu_dr_get_pgt_info()
1225 if (hop_addr == pgt_info->shadow_addr) in hl_mmu_dr_get_pgt_info()
1240 struct hl_device *hdev = ctx->hdev; in hl_mmu_dr_free_pgt_node()
1242 gen_pool_free(hdev->mmu_priv.dr.mmu_pgt_pool, pgt_info->phys_addr, in hl_mmu_dr_free_pgt_node()
1243 hdev->asic_prop.dmmu.hop_table_size); in hl_mmu_dr_free_pgt_node()
1244 hash_del(&pgt_info->node); in hl_mmu_dr_free_pgt_node()
1245 kfree((u64 *) (uintptr_t) pgt_info->shadow_addr); in hl_mmu_dr_free_pgt_node()
1251 return ctx->hdev->asic_prop.mmu_pgt_addr + in hl_mmu_dr_get_phys_hop0_addr()
1252 (ctx->asid * ctx->hdev->asic_prop.dmmu.hop_table_size); in hl_mmu_dr_get_phys_hop0_addr()
1257 return (u64) (uintptr_t) ctx->hdev->mmu_priv.dr.mmu_shadow_hop0 + in hl_mmu_dr_get_hop0_addr()
1258 (ctx->asid * ctx->hdev->asic_prop.dmmu.hop_table_size); in hl_mmu_dr_get_hop0_addr()
1263 u64 page_mask = ctx->hdev->asic_prop.dmmu.hop_table_size - 1; in hl_mmu_dr_get_phys_addr()
1269 phys_hop_addr = hl_mmu_dr_get_pgt_info(ctx, shadow_hop_addr)->phys_addr; in hl_mmu_dr_get_phys_addr()
1280 ctx->hdev->asic_funcs->write_pte(ctx->hdev, hl_mmu_dr_get_phys_addr(ctx, shadow_pte_addr), in hl_mmu_dr_write_pte()
1288 ctx->hdev->asic_funcs->write_pte(ctx->hdev, in hl_mmu_dr_write_final_pte()
1300 hl_mmu_dr_get_pgt_info(ctx, hop_addr)->num_of_ptes++; in hl_mmu_dr_get_pte()
1308 pgt_info->num_of_ptes--; in hl_mmu_dr_put_pte()
1314 num_of_ptes_left = pgt_info->num_of_ptes; in hl_mmu_dr_put_pte()
1323 struct hl_device *hdev = ctx->hdev; in hl_mmu_dr_alloc_hop()
1324 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_mmu_dr_alloc_hop()
1332 phys_addr = (u64) gen_pool_alloc(hdev->mmu_priv.dr.mmu_pgt_pool, in hl_mmu_dr_alloc_hop()
1333 prop->dmmu.hop_table_size); in hl_mmu_dr_alloc_hop()
1335 dev_err(hdev->dev, "failed to allocate page\n"); in hl_mmu_dr_alloc_hop()
1339 shadow_addr = (u64) (uintptr_t) kzalloc(prop->dmmu.hop_table_size, in hl_mmu_dr_alloc_hop()
1344 pgt_info->phys_addr = phys_addr; in hl_mmu_dr_alloc_hop()
1345 pgt_info->shadow_addr = shadow_addr; in hl_mmu_dr_alloc_hop()
1346 pgt_info->ctx = ctx; in hl_mmu_dr_alloc_hop()
1347 pgt_info->num_of_ptes = 0; in hl_mmu_dr_alloc_hop()
1348 hash_add(ctx->mmu_shadow_hash, &pgt_info->node, shadow_addr); in hl_mmu_dr_alloc_hop()
1353 gen_pool_free(hdev->mmu_priv.dr.mmu_pgt_pool, in hl_mmu_dr_alloc_hop()
1354 phys_addr, prop->dmmu.hop_table_size); in hl_mmu_dr_alloc_hop()
1377 ctx->hdev->asic_funcs->read_pte(ctx->hdev, hl_mmu_dr_get_phys_hop0_addr(ctx)); in hl_mmu_dr_flush()
1382 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_mmu_dr_init()
1385 hdev->mmu_priv.dr.mmu_pgt_pool = in hl_mmu_dr_init()
1386 gen_pool_create(__ffs(prop->dmmu.hop_table_size), -1); in hl_mmu_dr_init()
1388 if (!hdev->mmu_priv.dr.mmu_pgt_pool) { in hl_mmu_dr_init()
1389 dev_err(hdev->dev, "Failed to create page gen pool\n"); in hl_mmu_dr_init()
1390 return -ENOMEM; in hl_mmu_dr_init()
1393 rc = gen_pool_add(hdev->mmu_priv.dr.mmu_pgt_pool, prop->mmu_pgt_addr + in hl_mmu_dr_init()
1394 prop->dmmu.hop0_tables_total_size, in hl_mmu_dr_init()
1395 prop->dmmu.pgt_size - prop->dmmu.hop0_tables_total_size, in hl_mmu_dr_init()
1396 -1); in hl_mmu_dr_init()
1398 dev_err(hdev->dev, "Failed to add memory to page gen pool\n"); in hl_mmu_dr_init()
1402 hdev->mmu_priv.dr.mmu_shadow_hop0 = kvcalloc(prop->max_asid, in hl_mmu_dr_init()
1403 prop->dmmu.hop_table_size, GFP_KERNEL); in hl_mmu_dr_init()
1404 if (ZERO_OR_NULL_PTR(hdev->mmu_priv.dr.mmu_shadow_hop0)) { in hl_mmu_dr_init()
1405 rc = -ENOMEM; in hl_mmu_dr_init()
1409 /* MMU H/W init will be done in device hw_init() */ in hl_mmu_dr_init()
1414 gen_pool_destroy(hdev->mmu_priv.dr.mmu_pgt_pool); in hl_mmu_dr_init()
1421 /* MMU H/W fini was already done in device hw_fini() */ in hl_mmu_dr_fini()
1423 if (ZERO_OR_NULL_PTR(hdev->mmu_priv.dr.mmu_shadow_hop0)) in hl_mmu_dr_fini()
1426 kvfree(hdev->mmu_priv.dr.mmu_shadow_hop0); in hl_mmu_dr_fini()
1427 gen_pool_destroy(hdev->mmu_priv.dr.mmu_pgt_pool); in hl_mmu_dr_fini()
1431 * example if we fail during hard reset code at certain points in hl_mmu_dr_fini()
1433 hdev->mmu_priv.dr.mmu_shadow_hop0 = NULL; in hl_mmu_dr_fini()