Lines Matching +full:ctx +full:- +full:asid
1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2022 HabanaLabs, Ltd.
16 * hl_mmu_get_funcs() - get MMU functions structure
26 return &hdev->mmu_func[pgt_residency]; in hl_mmu_get_funcs()
31 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_is_dram_va()
33 return hl_mem_area_inside_range(virt_addr, prop->dmmu.page_size, in hl_is_dram_va()
34 prop->dmmu.start_addr, in hl_is_dram_va()
35 prop->dmmu.end_addr); in hl_is_dram_va()
39 * hl_mmu_init() - initialize the MMU module.
42 * Return: 0 for success, non-zero for failure.
46 int rc = -EOPNOTSUPP; in hl_mmu_init()
48 if (hdev->mmu_disable) in hl_mmu_init()
51 mutex_init(&hdev->mmu_lock); in hl_mmu_init()
53 if (hdev->mmu_func[MMU_DR_PGT].init != NULL) { in hl_mmu_init()
54 rc = hdev->mmu_func[MMU_DR_PGT].init(hdev); in hl_mmu_init()
59 if (hdev->mmu_func[MMU_HR_PGT].init != NULL) { in hl_mmu_init()
60 rc = hdev->mmu_func[MMU_HR_PGT].init(hdev); in hl_mmu_init()
68 if (hdev->mmu_func[MMU_DR_PGT].fini != NULL) in hl_mmu_init()
69 hdev->mmu_func[MMU_DR_PGT].fini(hdev); in hl_mmu_init()
75 * hl_mmu_fini() - release the MMU module.
79 * - Disable MMU in H/W.
80 * - Free the pgt_infos pool.
86 if (hdev->mmu_disable) in hl_mmu_fini()
89 if (hdev->mmu_func[MMU_DR_PGT].fini != NULL) in hl_mmu_fini()
90 hdev->mmu_func[MMU_DR_PGT].fini(hdev); in hl_mmu_fini()
92 if (hdev->mmu_func[MMU_HR_PGT].fini != NULL) in hl_mmu_fini()
93 hdev->mmu_func[MMU_HR_PGT].fini(hdev); in hl_mmu_fini()
95 mutex_destroy(&hdev->mmu_lock); in hl_mmu_fini()
99 * hl_mmu_ctx_init() - initialize a context for using the MMU module.
100 * @ctx: pointer to the context structure to initialize.
104 * Return: 0 on success, non-zero otherwise.
106 int hl_mmu_ctx_init(struct hl_ctx *ctx) in hl_mmu_ctx_init() argument
108 struct hl_device *hdev = ctx->hdev; in hl_mmu_ctx_init()
109 int rc = -EOPNOTSUPP; in hl_mmu_ctx_init()
111 if (hdev->mmu_disable) in hl_mmu_ctx_init()
114 if (hdev->mmu_func[MMU_DR_PGT].ctx_init != NULL) { in hl_mmu_ctx_init()
115 rc = hdev->mmu_func[MMU_DR_PGT].ctx_init(ctx); in hl_mmu_ctx_init()
120 if (hdev->mmu_func[MMU_HR_PGT].ctx_init != NULL) { in hl_mmu_ctx_init()
121 rc = hdev->mmu_func[MMU_HR_PGT].ctx_init(ctx); in hl_mmu_ctx_init()
129 if (hdev->mmu_func[MMU_DR_PGT].fini != NULL) in hl_mmu_ctx_init()
130 hdev->mmu_func[MMU_DR_PGT].fini(hdev); in hl_mmu_ctx_init()
136 * hl_mmu_ctx_fini - disable a ctx from using the mmu module
138 * @ctx: pointer to the context structure
141 * - Free any pgts which were not freed yet
142 * - Free the mutex
143 * - Free DRAM default page mapping hops
145 void hl_mmu_ctx_fini(struct hl_ctx *ctx) in hl_mmu_ctx_fini() argument
147 struct hl_device *hdev = ctx->hdev; in hl_mmu_ctx_fini()
149 if (hdev->mmu_disable) in hl_mmu_ctx_fini()
152 if (hdev->mmu_func[MMU_DR_PGT].ctx_fini != NULL) in hl_mmu_ctx_fini()
153 hdev->mmu_func[MMU_DR_PGT].ctx_fini(ctx); in hl_mmu_ctx_fini()
155 if (hdev->mmu_func[MMU_HR_PGT].ctx_fini != NULL) in hl_mmu_ctx_fini()
156 hdev->mmu_func[MMU_HR_PGT].ctx_fini(ctx); in hl_mmu_ctx_fini()
160 * hl_mmu_get_real_page_size - get real page size to use in map/unmap operation
172 * 1. it shall not be called directly- only from mmu_func structure instance
180 * size is bigger, we break it to sub-pages and map them separately. in hl_mmu_get_real_page_size()
182 if ((page_size % mmu_prop->page_size) == 0) { in hl_mmu_get_real_page_size()
183 *real_page_size = mmu_prop->page_size; in hl_mmu_get_real_page_size()
187 dev_err(hdev->dev, "page size of %u is not %uKB aligned, can't map\n", in hl_mmu_get_real_page_size()
188 page_size, mmu_prop->page_size >> 10); in hl_mmu_get_real_page_size()
190 return -EFAULT; in hl_mmu_get_real_page_size()
196 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_mmu_get_prop()
199 return &prop->dmmu; in hl_mmu_get_prop()
200 else if ((page_size % prop->pmmu_huge.page_size) == 0) in hl_mmu_get_prop()
201 return &prop->pmmu_huge; in hl_mmu_get_prop()
203 return &prop->pmmu; in hl_mmu_get_prop()
207 * hl_mmu_unmap_page - unmaps a virtual addr
209 * @ctx: pointer to the context structure
215 * - Check that the virt addr is mapped
216 * - Unmap the virt addr and frees pgts if possible
217 * - Returns 0 on success, -EINVAL if the given addr is not mapped
227 int hl_mmu_unmap_page(struct hl_ctx *ctx, u64 virt_addr, u32 page_size, bool flush_pte) in hl_mmu_unmap_page() argument
229 struct hl_device *hdev = ctx->hdev; in hl_mmu_unmap_page()
237 if (hdev->mmu_disable) in hl_mmu_unmap_page()
243 pgt_residency = mmu_prop->host_resident ? MMU_HR_PGT : MMU_DR_PGT; in hl_mmu_unmap_page()
246 rc = hdev->asic_funcs->mmu_get_real_page_size(hdev, mmu_prop, page_size, &real_page_size, in hl_mmu_unmap_page()
255 rc = mmu_funcs->unmap(ctx, real_virt_addr, is_dram_addr); in hl_mmu_unmap_page()
263 mmu_funcs->flush(ctx); in hl_mmu_unmap_page()
266 trace_habanalabs_mmu_unmap(&hdev->pdev->dev, virt_addr, 0, page_size, flush_pte); in hl_mmu_unmap_page()
272 * hl_mmu_map_page - maps a virtual addr to physical addr
274 * @ctx: pointer to the context structure
281 * - Check that the virt addr is not mapped
282 * - Allocate pgts as necessary in order to map the virt addr to the phys
283 * - Returns 0 on success, -EINVAL if addr is already mapped, or -ENOMEM.
293 int hl_mmu_map_page(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, u32 page_size, in hl_mmu_map_page() argument
297 struct hl_device *hdev = ctx->hdev; in hl_mmu_map_page()
305 if (hdev->mmu_disable) in hl_mmu_map_page()
311 pgt_residency = mmu_prop->host_resident ? MMU_HR_PGT : MMU_DR_PGT; in hl_mmu_map_page()
314 rc = hdev->asic_funcs->mmu_get_real_page_size(hdev, mmu_prop, page_size, &real_page_size, in hl_mmu_map_page()
325 ((hdev->asic_funcs->scramble_addr(hdev, phys_addr) & in hl_mmu_map_page()
326 (mmu_prop->page_size - 1)) || in hl_mmu_map_page()
327 (hdev->asic_funcs->scramble_addr(hdev, virt_addr) & in hl_mmu_map_page()
328 (mmu_prop->page_size - 1)))) || in hl_mmu_map_page()
329 (!is_dram_addr && ((phys_addr & (real_page_size - 1)) || in hl_mmu_map_page()
330 (virt_addr & (real_page_size - 1))))) in hl_mmu_map_page()
331 dev_crit(hdev->dev, in hl_mmu_map_page()
340 rc = mmu_funcs->map(ctx, real_virt_addr, real_phys_addr, real_page_size, in hl_mmu_map_page()
351 mmu_funcs->flush(ctx); in hl_mmu_map_page()
353 trace_habanalabs_mmu_map(&hdev->pdev->dev, virt_addr, phys_addr, page_size, flush_pte); in hl_mmu_map_page()
360 if (mmu_funcs->unmap(ctx, real_virt_addr, is_dram_addr)) in hl_mmu_map_page()
361 dev_warn_ratelimited(hdev->dev, in hl_mmu_map_page()
367 mmu_funcs->flush(ctx); in hl_mmu_map_page()
373 * hl_mmu_map_contiguous - implements a wrapper for hl_mmu_map_page
376 * @ctx: pointer to the context structure
382 int hl_mmu_map_contiguous(struct hl_ctx *ctx, u64 virt_addr, in hl_mmu_map_contiguous() argument
385 struct hl_device *hdev = ctx->hdev; in hl_mmu_map_contiguous()
386 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_mmu_map_contiguous()
393 prop->dmmu.start_addr, prop->dmmu.end_addr)) in hl_mmu_map_contiguous()
394 page_size = prop->dmmu.page_size; in hl_mmu_map_contiguous()
396 prop->pmmu.start_addr, prop->pmmu.end_addr)) in hl_mmu_map_contiguous()
397 page_size = prop->pmmu.page_size; in hl_mmu_map_contiguous()
399 prop->pmmu_huge.start_addr, prop->pmmu_huge.end_addr)) in hl_mmu_map_contiguous()
400 page_size = prop->pmmu_huge.page_size; in hl_mmu_map_contiguous()
402 return -EINVAL; in hl_mmu_map_contiguous()
408 rc = hl_mmu_map_page(ctx, curr_va, curr_pa, page_size, in hl_mmu_map_contiguous()
411 dev_err(hdev->dev, in hl_mmu_map_contiguous()
414 /* last mapping failed so don't try to unmap it - reduce off by page_size */ in hl_mmu_map_contiguous()
415 off -= page_size; in hl_mmu_map_contiguous()
423 for (; off >= 0 ; off -= page_size) { in hl_mmu_map_contiguous()
425 flush_pte = (off - (s32) page_size) < 0; in hl_mmu_map_contiguous()
426 if (hl_mmu_unmap_page(ctx, curr_va, page_size, flush_pte)) in hl_mmu_map_contiguous()
427 dev_warn_ratelimited(hdev->dev, in hl_mmu_map_contiguous()
435 * hl_mmu_unmap_contiguous - implements a wrapper for hl_mmu_unmap_page
438 * @ctx: pointer to the context structure
443 int hl_mmu_unmap_contiguous(struct hl_ctx *ctx, u64 virt_addr, u32 size) in hl_mmu_unmap_contiguous() argument
445 struct hl_device *hdev = ctx->hdev; in hl_mmu_unmap_contiguous()
446 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_mmu_unmap_contiguous()
453 prop->dmmu.start_addr, prop->dmmu.end_addr)) in hl_mmu_unmap_contiguous()
454 page_size = prop->dmmu.page_size; in hl_mmu_unmap_contiguous()
456 prop->pmmu.start_addr, prop->pmmu.end_addr)) in hl_mmu_unmap_contiguous()
457 page_size = prop->pmmu.page_size; in hl_mmu_unmap_contiguous()
459 prop->pmmu_huge.start_addr, prop->pmmu_huge.end_addr)) in hl_mmu_unmap_contiguous()
460 page_size = prop->pmmu_huge.page_size; in hl_mmu_unmap_contiguous()
462 return -EINVAL; in hl_mmu_unmap_contiguous()
467 rc = hl_mmu_unmap_page(ctx, curr_va, page_size, flush_pte); in hl_mmu_unmap_contiguous()
469 dev_warn_ratelimited(hdev->dev, in hl_mmu_unmap_contiguous()
476 static void hl_mmu_pa_page_with_offset(struct hl_ctx *ctx, u64 virt_addr, in hl_mmu_pa_page_with_offset() argument
480 struct asic_fixed_properties *prop = &ctx->hdev->asic_prop; in hl_mmu_pa_page_with_offset()
485 if (hops->unscrambled_paddr) in hl_mmu_pa_page_with_offset()
486 tmp_phys_addr = hops->unscrambled_paddr; in hl_mmu_pa_page_with_offset()
488 tmp_phys_addr = hops->hop_info[hops->used_hops - 1].hop_pte_val; in hl_mmu_pa_page_with_offset()
490 if (hops->range_type == HL_VA_RANGE_TYPE_HOST_HUGE) in hl_mmu_pa_page_with_offset()
491 mmu_prop = &prop->pmmu_huge; in hl_mmu_pa_page_with_offset()
492 else if (hops->range_type == HL_VA_RANGE_TYPE_HOST) in hl_mmu_pa_page_with_offset()
493 mmu_prop = &prop->pmmu; in hl_mmu_pa_page_with_offset()
495 mmu_prop = &prop->dmmu; in hl_mmu_pa_page_with_offset()
497 if ((hops->range_type == HL_VA_RANGE_TYPE_DRAM) && in hl_mmu_pa_page_with_offset()
498 !is_power_of_2(prop->dram_page_size)) { in hl_mmu_pa_page_with_offset()
509 dram_page_size = prop->dram_page_size; in hl_mmu_pa_page_with_offset()
510 dram_base = prop->dram_base_address; in hl_mmu_pa_page_with_offset()
511 abs_phys_addr = tmp_phys_addr - dram_base; in hl_mmu_pa_page_with_offset()
512 abs_virt_addr = virt_addr - dram_base; in hl_mmu_pa_page_with_offset()
524 hop_shift = mmu_prop->hop_shifts[hops->used_hops - 1]; in hl_mmu_pa_page_with_offset()
525 offset_mask = (1ull << hop_shift) - 1; in hl_mmu_pa_page_with_offset()
532 int hl_mmu_va_to_pa(struct hl_ctx *ctx, u64 virt_addr, u64 *phys_addr) in hl_mmu_va_to_pa() argument
539 rc = hl_mmu_get_tlb_info(ctx, virt_addr, &hops); in hl_mmu_va_to_pa()
543 hl_mmu_pa_page_with_offset(ctx, virt_addr, &hops, phys_addr); in hl_mmu_va_to_pa()
548 int hl_mmu_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr, in hl_mmu_get_tlb_info() argument
551 struct hl_device *hdev = ctx->hdev; in hl_mmu_get_tlb_info()
558 if (hdev->mmu_disable) in hl_mmu_get_tlb_info()
559 return -EOPNOTSUPP; in hl_mmu_get_tlb_info()
561 prop = &hdev->asic_prop; in hl_mmu_get_tlb_info()
562 hops->scrambled_vaddr = virt_addr; /* assume no scrambling */ in hl_mmu_get_tlb_info()
564 is_dram_addr = hl_mem_area_inside_range(virt_addr, prop->dmmu.page_size, in hl_mmu_get_tlb_info()
565 prop->dmmu.start_addr, in hl_mmu_get_tlb_info()
566 prop->dmmu.end_addr); in hl_mmu_get_tlb_info()
568 /* host-residency is the same in PMMU and PMMU huge, no need to distinguish here */ in hl_mmu_get_tlb_info()
569 mmu_prop = is_dram_addr ? &prop->dmmu : &prop->pmmu; in hl_mmu_get_tlb_info()
570 pgt_residency = mmu_prop->host_resident ? MMU_HR_PGT : MMU_DR_PGT; in hl_mmu_get_tlb_info()
573 mutex_lock(&hdev->mmu_lock); in hl_mmu_get_tlb_info()
574 rc = mmu_funcs->get_tlb_info(ctx, virt_addr, hops); in hl_mmu_get_tlb_info()
575 mutex_unlock(&hdev->mmu_lock); in hl_mmu_get_tlb_info()
581 if (hops->unscrambled_paddr) in hl_mmu_get_tlb_info()
582 hl_mmu_pa_page_with_offset(ctx, virt_addr, hops, &hops->unscrambled_paddr); in hl_mmu_get_tlb_info()
589 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_mmu_if_set_funcs()
591 if (hdev->mmu_disable) in hl_mmu_if_set_funcs()
594 switch (hdev->asic_type) { in hl_mmu_if_set_funcs()
598 hl_mmu_v1_set_funcs(hdev, &hdev->mmu_func[MMU_DR_PGT]); in hl_mmu_if_set_funcs()
604 hl_mmu_v2_set_funcs(hdev, &hdev->mmu_func[MMU_DR_PGT]); in hl_mmu_if_set_funcs()
605 if (prop->pmmu.host_resident) in hl_mmu_if_set_funcs()
606 hl_mmu_v2_hr_set_funcs(hdev, &hdev->mmu_func[MMU_HR_PGT]); in hl_mmu_if_set_funcs()
609 dev_err(hdev->dev, "Unrecognized ASIC type %d\n", in hl_mmu_if_set_funcs()
610 hdev->asic_type); in hl_mmu_if_set_funcs()
611 return -EOPNOTSUPP; in hl_mmu_if_set_funcs()
618 * hl_mmu_scramble_addr() - The generic mmu address scrambling routine.
630 * hl_mmu_descramble_addr() - The generic mmu address descrambling
635 * Return: The un-scrambled address.
646 rc = hdev->asic_funcs->mmu_invalidate_cache(hdev, is_hard, flags); in hl_mmu_invalidate_cache()
648 dev_err_ratelimited(hdev->dev, in hl_mmu_invalidate_cache()
650 dev_name(&hdev->pdev->dev), in hl_mmu_invalidate_cache()
657 u32 flags, u32 asid, u64 va, u64 size) in hl_mmu_invalidate_cache_range() argument
661 rc = hdev->asic_funcs->mmu_invalidate_cache_range(hdev, is_hard, flags, in hl_mmu_invalidate_cache_range()
662 asid, va, size); in hl_mmu_invalidate_cache_range()
664 dev_err_ratelimited(hdev->dev, in hl_mmu_invalidate_cache_range()
666 dev_name(&hdev->pdev->dev), flags == VM_TYPE_USERPTR ? "PMMU" : "HMMU", in hl_mmu_invalidate_cache_range()
675 struct hl_ctx *ctx = pfw->ctx; in hl_mmu_prefetch_work_function() local
676 struct hl_device *hdev = ctx->hdev; in hl_mmu_prefetch_work_function()
681 mutex_lock(&hdev->mmu_lock); in hl_mmu_prefetch_work_function()
683 hdev->asic_funcs->mmu_prefetch_cache_range(ctx, pfw->flags, pfw->asid, pfw->va, pfw->size); in hl_mmu_prefetch_work_function()
685 mutex_unlock(&hdev->mmu_lock); in hl_mmu_prefetch_work_function()
689 * context was taken in the common mmu prefetch function- see comment there about in hl_mmu_prefetch_work_function()
692 hl_ctx_put(ctx); in hl_mmu_prefetch_work_function()
696 int hl_mmu_prefetch_cache_range(struct hl_ctx *ctx, u32 flags, u32 asid, u64 va, u64 size) in hl_mmu_prefetch_cache_range() argument
702 return -ENOMEM; in hl_mmu_prefetch_cache_range()
704 INIT_WORK(&handle_prefetch_work->prefetch_work, hl_mmu_prefetch_work_function); in hl_mmu_prefetch_cache_range()
705 handle_prefetch_work->ctx = ctx; in hl_mmu_prefetch_cache_range()
706 handle_prefetch_work->va = va; in hl_mmu_prefetch_cache_range()
707 handle_prefetch_work->size = size; in hl_mmu_prefetch_cache_range()
708 handle_prefetch_work->flags = flags; in hl_mmu_prefetch_cache_range()
709 handle_prefetch_work->asid = asid; in hl_mmu_prefetch_cache_range()
715 hl_ctx_get(ctx); in hl_mmu_prefetch_cache_range()
716 queue_work(ctx->hdev->prefetch_wq, &handle_prefetch_work->prefetch_work); in hl_mmu_prefetch_cache_range()
721 u64 hl_mmu_get_next_hop_addr(struct hl_ctx *ctx, u64 curr_pte) in hl_mmu_get_next_hop_addr() argument
727 * hl_mmu_get_hop_pte_phys_addr() - extract PTE address from HOP
728 * @ctx: pointer to the context structure to initialize.
736 u64 hl_mmu_get_hop_pte_phys_addr(struct hl_ctx *ctx, struct hl_mmu_properties *mmu_prop, in hl_mmu_get_hop_pte_phys_addr() argument
741 if (hop_idx >= mmu_prop->num_hops) { in hl_mmu_get_hop_pte_phys_addr()
742 dev_err_ratelimited(ctx->hdev->dev, "Invalid hop index %d\n", hop_idx); in hl_mmu_get_hop_pte_phys_addr()
746 shift = mmu_prop->hop_shifts[hop_idx]; in hl_mmu_get_hop_pte_phys_addr()
747 mask = mmu_prop->hop_masks[hop_idx]; in hl_mmu_get_hop_pte_phys_addr()
749 return hop_addr + ctx->hdev->asic_prop.mmu_pte_size * ((virt_addr & mask) >> shift); in hl_mmu_get_hop_pte_phys_addr()
758 hl_asic_dma_free_coherent(hdev, (chunk->end_addr - chunk->start_addr) + 1, in mmu_dma_mem_free_from_chunk()
759 (void *)chunk->start_addr, chunk->phys_addr); in mmu_dma_mem_free_from_chunk()
762 void hl_mmu_hr_flush(struct hl_ctx *ctx) in hl_mmu_hr_flush() argument
769 * hl_mmu_hr_pool_destroy() - destroy genpool
775 * - free entries allocated for shadow HOP0
776 * - free pool chunks
777 * - free pool
782 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_mmu_hr_pool_destroy()
783 struct gen_pool **pool = &hr_priv->mmu_pgt_pool; in hl_mmu_hr_pool_destroy()
785 int asid; in hl_mmu_hr_pool_destroy() local
791 if (hr_priv->mmu_asid_hop0) { in hl_mmu_hr_pool_destroy()
792 for (asid = 0 ; asid < prop->max_asid ; asid++) { in hl_mmu_hr_pool_destroy()
793 hop0_pgt = &hr_priv->mmu_asid_hop0[asid]; in hl_mmu_hr_pool_destroy()
794 if (ZERO_OR_NULL_PTR(hop0_pgt->virt_addr)) in hl_mmu_hr_pool_destroy()
797 gen_pool_free(*pool, (uintptr_t) hop0_pgt->virt_addr, hop_table_size); in hl_mmu_hr_pool_destroy()
812 * hl_mmu_hr_init() - initialize the MMU module.
818 * @return 0 on success otherwise non-zero error code
821 * - Create a pool of pages for pgt_infos.
822 * - Create a shadow table for pgt
827 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_mmu_hr_init()
838 * allocations with higher-order alignment restrictions in hl_mmu_hr_init()
840 hr_priv->mmu_pgt_pool = gen_pool_create(PAGE_SHIFT, -1); in hl_mmu_hr_init()
841 if (ZERO_OR_NULL_PTR(hr_priv->mmu_pgt_pool)) { in hl_mmu_hr_init()
842 dev_err(hdev->dev, "Failed to create hr page pool\n"); in hl_mmu_hr_init()
843 return -ENOMEM; in hl_mmu_hr_init()
846 hr_priv->mmu_asid_hop0 = kvcalloc(prop->max_asid, sizeof(struct pgt_info), GFP_KERNEL); in hl_mmu_hr_init()
847 if (ZERO_OR_NULL_PTR(hr_priv->mmu_asid_hop0)) { in hl_mmu_hr_init()
848 dev_err(hdev->dev, "Failed to allocate hr-mmu hop0 table\n"); in hl_mmu_hr_init()
849 rc = -ENOMEM; in hl_mmu_hr_init()
858 dev_err(hdev->dev, in hl_mmu_hr_init()
859 "Failed to allocate memory for host-resident page pool\n"); in hl_mmu_hr_init()
860 rc = -ENOMEM; in hl_mmu_hr_init()
864 rc = gen_pool_add_virt(hr_priv->mmu_pgt_pool, virt_addr, (phys_addr_t) dma_addr, in hl_mmu_hr_init()
865 pool_chunk_size, -1); in hl_mmu_hr_init()
867 dev_err(hdev->dev, "Failed to fill host-resident page pool\n"); in hl_mmu_hr_init()
872 for (i = 0 ; i < prop->max_asid ; i++) { in hl_mmu_hr_init()
873 hop0_pgt = &hr_priv->mmu_asid_hop0[i]; in hl_mmu_hr_init()
874 hop0_pgt->virt_addr = (uintptr_t) in hl_mmu_hr_init()
875 gen_pool_dma_zalloc_align(hr_priv->mmu_pgt_pool, in hl_mmu_hr_init()
877 (dma_addr_t *) &hop0_pgt->phys_addr, in hl_mmu_hr_init()
879 if (!hop0_pgt->virt_addr) { in hl_mmu_hr_init()
880 dev_err(hdev->dev, "Failed to allocate HOP from pgt pool\n"); in hl_mmu_hr_init()
881 rc = -ENOMEM; in hl_mmu_hr_init()
892 if (!ZERO_OR_NULL_PTR(hr_priv->mmu_asid_hop0)) in hl_mmu_hr_init()
893 kvfree(hr_priv->mmu_asid_hop0); in hl_mmu_hr_init()
899 * hl_mmu_hr_fini() - release the MMU module.
905 * - Disable MMU in H/W.
906 * - Free the pgt_infos pool.
916 if (!ZERO_OR_NULL_PTR(hr_priv->mmu_asid_hop0)) { in hl_mmu_hr_fini()
917 kvfree(hr_priv->mmu_asid_hop0); in hl_mmu_hr_fini()
923 hr_priv->mmu_asid_hop0 = NULL; in hl_mmu_hr_fini()
928 * hl_mmu_hr_free_hop_remove_pgt() - free HOP and remove PGT from hash
936 gen_pool_free(hr_priv->mmu_pgt_pool, pgt_info->virt_addr, hop_table_size); in hl_mmu_hr_free_hop_remove_pgt()
937 hash_del(&pgt_info->node); in hl_mmu_hr_free_hop_remove_pgt()
942 * hl_mmu_hr_pte_phys_to_virt() - translate PTE phys addr to virt addr
943 * @ctx: pointer to the context structure
953 u64 hl_mmu_hr_pte_phys_to_virt(struct hl_ctx *ctx, struct pgt_info *pgt, in hl_mmu_hr_pte_phys_to_virt() argument
956 u64 page_mask = (hop_table_size - 1); in hl_mmu_hr_pte_phys_to_virt()
959 return pgt->virt_addr + pte_offset; in hl_mmu_hr_pte_phys_to_virt()
963 * hl_mmu_hr_write_pte() - write HR PTE
964 * @ctx: pointer to the context structure
970 void hl_mmu_hr_write_pte(struct hl_ctx *ctx, struct pgt_info *pgt_info, u64 phys_pte_addr, in hl_mmu_hr_write_pte() argument
977 u64 virt_addr = hl_mmu_hr_pte_phys_to_virt(ctx, pgt_info, phys_pte_addr, hop_table_size); in hl_mmu_hr_write_pte()
983 * hl_mmu_hr_clear_pte() - clear HR PTE
984 * @ctx: pointer to the context structure
989 void hl_mmu_hr_clear_pte(struct hl_ctx *ctx, struct pgt_info *pgt_info, u64 phys_pte_addr, in hl_mmu_hr_clear_pte() argument
993 hl_mmu_hr_write_pte(ctx, pgt_info, phys_pte_addr, 0, hop_table_size); in hl_mmu_hr_clear_pte()
997 * hl_mmu_hr_put_pte() - put HR PTE and remove it if necessary (no more PTEs)
998 * @ctx: pointer to the context structure
1005 int hl_mmu_hr_put_pte(struct hl_ctx *ctx, struct pgt_info *pgt_info, in hl_mmu_hr_put_pte() argument
1011 pgt_info->num_of_ptes--; in hl_mmu_hr_put_pte()
1017 num_of_ptes_left = pgt_info->num_of_ptes; in hl_mmu_hr_put_pte()
1025 * hl_mmu_hr_get_pte() - increase PGT PTE count
1026 * @ctx: pointer to the context structure
1030 void hl_mmu_hr_get_pte(struct hl_ctx *ctx, struct hl_hr_mmu_funcs *hr_func, u64 phys_hop_addr) in hl_mmu_hr_get_pte() argument
1032 hr_func->get_pgt_info(ctx, phys_hop_addr)->num_of_ptes++; in hl_mmu_hr_get_pte()
1036 * hl_mmu_hr_get_next_hop_pgt_info() - get pgt_info structure for the next HOP
1037 * @ctx: pointer to the context structure.
1043 struct pgt_info *hl_mmu_hr_get_next_hop_pgt_info(struct hl_ctx *ctx, in hl_mmu_hr_get_next_hop_pgt_info() argument
1047 u64 next_hop_phys_addr = hl_mmu_get_next_hop_addr(ctx, curr_pte); in hl_mmu_hr_get_next_hop_pgt_info()
1052 return hr_func->get_pgt_info(ctx, next_hop_phys_addr); in hl_mmu_hr_get_next_hop_pgt_info()
1056 * hl_mmu_hr_alloc_hop() - allocate HOP
1057 * @ctx: pointer to the context structure.
1064 struct pgt_info *hl_mmu_hr_alloc_hop(struct hl_ctx *ctx, struct hl_mmu_hr_priv *hr_priv, in hl_mmu_hr_alloc_hop() argument
1068 struct hl_device *hdev = ctx->hdev; in hl_mmu_hr_alloc_hop()
1079 virt_addr = gen_pool_dma_zalloc_align(hr_priv->mmu_pgt_pool, in hl_mmu_hr_alloc_hop()
1080 mmu_prop->hop_table_size, in hl_mmu_hr_alloc_hop()
1082 mmu_prop->hop_table_size); in hl_mmu_hr_alloc_hop()
1086 /* No memory in pool - get some and try again */ in hl_mmu_hr_alloc_hop()
1092 if (gen_pool_add_virt(hr_priv->mmu_pgt_pool, (unsigned long)virt_addr, in hl_mmu_hr_alloc_hop()
1093 phys_addr, SZ_2M, -1)) { in hl_mmu_hr_alloc_hop()
1101 dev_err(hdev->dev, "failed to allocate page\n"); in hl_mmu_hr_alloc_hop()
1105 pgt_info->phys_addr = phys_addr; in hl_mmu_hr_alloc_hop()
1106 pgt_info->shadow_addr = (unsigned long) NULL; in hl_mmu_hr_alloc_hop()
1107 pgt_info->virt_addr = (unsigned long)virt_addr; in hl_mmu_hr_alloc_hop()
1108 pgt_info->ctx = ctx; in hl_mmu_hr_alloc_hop()
1109 pgt_info->num_of_ptes = 0; in hl_mmu_hr_alloc_hop()
1110 hr_func->add_pgt_info(ctx, pgt_info, phys_addr); in hl_mmu_hr_alloc_hop()
1121 * hl_mmu_hr_get_alloc_next_hop() - get the next HOP, allocate it if it does not exist
1122 * @ctx: pointer to the context structure.
1131 struct pgt_info *hl_mmu_hr_get_alloc_next_hop(struct hl_ctx *ctx, in hl_mmu_hr_get_alloc_next_hop() argument
1137 u64 hop_addr = hl_mmu_get_next_hop_addr(ctx, curr_pte); in hl_mmu_hr_get_alloc_next_hop()
1140 return hr_func->get_pgt_info(ctx, hop_addr); in hl_mmu_hr_get_alloc_next_hop()
1143 return hl_mmu_hr_alloc_hop(ctx, hr_priv, hr_func, mmu_prop); in hl_mmu_hr_get_alloc_next_hop()
1147 * hl_mmu_hr_get_tlb_info() - get the TLB info (info for a specific mapping)
1148 * @ctx: pointer to the context structure.
1155 int hl_mmu_hr_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr, struct hl_mmu_hop_info *hops, in hl_mmu_hr_get_tlb_info() argument
1160 struct hl_device *hdev = ctx->hdev; in hl_mmu_hr_get_tlb_info()
1165 rc = hr_func->get_tlb_mapping_params(hdev, &mmu_prop, hops, virt_addr, &is_huge); in hl_mmu_hr_get_tlb_info()
1169 used_hops = mmu_prop->num_hops; in hl_mmu_hr_get_tlb_info()
1173 used_hops--; in hl_mmu_hr_get_tlb_info()
1175 hops->scrambled_vaddr = hdev->asic_funcs->scramble_addr(hdev, virt_addr); in hl_mmu_hr_get_tlb_info()
1179 hops_pgt_info[i] = hr_func->get_hop0_pgt_info(ctx); in hl_mmu_hr_get_tlb_info()
1181 hops_pgt_info[i] = hl_mmu_hr_get_next_hop_pgt_info(ctx, hr_func, in hl_mmu_hr_get_tlb_info()
1182 hops->hop_info[i - 1].hop_pte_val); in hl_mmu_hr_get_tlb_info()
1185 return -EFAULT; in hl_mmu_hr_get_tlb_info()
1187 hops->hop_info[i].hop_addr = hops_pgt_info[i]->phys_addr; in hl_mmu_hr_get_tlb_info()
1188 hops->hop_info[i].hop_pte_addr = in hl_mmu_hr_get_tlb_info()
1189 hl_mmu_get_hop_pte_phys_addr(ctx, mmu_prop, i, in hl_mmu_hr_get_tlb_info()
1190 hops->hop_info[i].hop_addr, in hl_mmu_hr_get_tlb_info()
1191 hops->scrambled_vaddr); in hl_mmu_hr_get_tlb_info()
1192 hops->hop_info[i].hop_pte_val = *(u64 *) (uintptr_t) in hl_mmu_hr_get_tlb_info()
1193 hl_mmu_hr_pte_phys_to_virt(ctx, hops_pgt_info[i], in hl_mmu_hr_get_tlb_info()
1194 hops->hop_info[i].hop_pte_addr, in hl_mmu_hr_get_tlb_info()
1195 mmu_prop->hop_table_size); in hl_mmu_hr_get_tlb_info()
1197 if (!(hops->hop_info[i].hop_pte_val & PAGE_PRESENT_MASK)) in hl_mmu_hr_get_tlb_info()
1198 return -EFAULT; in hl_mmu_hr_get_tlb_info()
1200 if (hops->hop_info[i].hop_pte_val & mmu_prop->last_mask) in hl_mmu_hr_get_tlb_info()
1205 if (i == mmu_prop->num_hops) in hl_mmu_hr_get_tlb_info()
1206 return -EFAULT; in hl_mmu_hr_get_tlb_info()
1208 if (hops->scrambled_vaddr != virt_addr) in hl_mmu_hr_get_tlb_info()
1209 hops->unscrambled_paddr = hdev->asic_funcs->descramble_addr in hl_mmu_hr_get_tlb_info()
1210 (hdev, hops->hop_info[i].hop_pte_val); in hl_mmu_hr_get_tlb_info()
1212 hops->unscrambled_paddr = hops->hop_info[i].hop_pte_val; in hl_mmu_hr_get_tlb_info()
1214 hops->used_hops = i + 1; in hl_mmu_hr_get_tlb_info()
1219 struct pgt_info *hl_mmu_dr_get_pgt_info(struct hl_ctx *ctx, u64 hop_addr) in hl_mmu_dr_get_pgt_info() argument
1223 hash_for_each_possible(ctx->mmu_shadow_hash, pgt_info, node, in hl_mmu_dr_get_pgt_info()
1225 if (hop_addr == pgt_info->shadow_addr) in hl_mmu_dr_get_pgt_info()
1231 void hl_mmu_dr_free_hop(struct hl_ctx *ctx, u64 hop_addr) in hl_mmu_dr_free_hop() argument
1233 struct pgt_info *pgt_info = hl_mmu_dr_get_pgt_info(ctx, hop_addr); in hl_mmu_dr_free_hop()
1235 hl_mmu_dr_free_pgt_node(ctx, pgt_info); in hl_mmu_dr_free_hop()
1238 void hl_mmu_dr_free_pgt_node(struct hl_ctx *ctx, struct pgt_info *pgt_info) in hl_mmu_dr_free_pgt_node() argument
1240 struct hl_device *hdev = ctx->hdev; in hl_mmu_dr_free_pgt_node()
1242 gen_pool_free(hdev->mmu_priv.dr.mmu_pgt_pool, pgt_info->phys_addr, in hl_mmu_dr_free_pgt_node()
1243 hdev->asic_prop.dmmu.hop_table_size); in hl_mmu_dr_free_pgt_node()
1244 hash_del(&pgt_info->node); in hl_mmu_dr_free_pgt_node()
1245 kfree((u64 *) (uintptr_t) pgt_info->shadow_addr); in hl_mmu_dr_free_pgt_node()
1249 u64 hl_mmu_dr_get_phys_hop0_addr(struct hl_ctx *ctx) in hl_mmu_dr_get_phys_hop0_addr() argument
1251 return ctx->hdev->asic_prop.mmu_pgt_addr + in hl_mmu_dr_get_phys_hop0_addr()
1252 (ctx->asid * ctx->hdev->asic_prop.dmmu.hop_table_size); in hl_mmu_dr_get_phys_hop0_addr()
1255 u64 hl_mmu_dr_get_hop0_addr(struct hl_ctx *ctx) in hl_mmu_dr_get_hop0_addr() argument
1257 return (u64) (uintptr_t) ctx->hdev->mmu_priv.dr.mmu_shadow_hop0 + in hl_mmu_dr_get_hop0_addr()
1258 (ctx->asid * ctx->hdev->asic_prop.dmmu.hop_table_size); in hl_mmu_dr_get_hop0_addr()
1261 u64 hl_mmu_dr_get_phys_addr(struct hl_ctx *ctx, u64 shadow_addr) in hl_mmu_dr_get_phys_addr() argument
1263 u64 page_mask = ctx->hdev->asic_prop.dmmu.hop_table_size - 1; in hl_mmu_dr_get_phys_addr()
1268 if (shadow_hop_addr != hl_mmu_dr_get_hop0_addr(ctx)) in hl_mmu_dr_get_phys_addr()
1269 phys_hop_addr = hl_mmu_dr_get_pgt_info(ctx, shadow_hop_addr)->phys_addr; in hl_mmu_dr_get_phys_addr()
1271 phys_hop_addr = hl_mmu_dr_get_phys_hop0_addr(ctx); in hl_mmu_dr_get_phys_addr()
1276 void hl_mmu_dr_write_pte(struct hl_ctx *ctx, u64 shadow_pte_addr, u64 val) in hl_mmu_dr_write_pte() argument
1278 u64 phys_val = hl_mmu_dr_get_phys_addr(ctx, val); in hl_mmu_dr_write_pte()
1280 ctx->hdev->asic_funcs->write_pte(ctx->hdev, hl_mmu_dr_get_phys_addr(ctx, shadow_pte_addr), in hl_mmu_dr_write_pte()
1286 void hl_mmu_dr_write_final_pte(struct hl_ctx *ctx, u64 shadow_pte_addr, u64 val) in hl_mmu_dr_write_final_pte() argument
1288 ctx->hdev->asic_funcs->write_pte(ctx->hdev, in hl_mmu_dr_write_final_pte()
1289 hl_mmu_dr_get_phys_addr(ctx, shadow_pte_addr), val); in hl_mmu_dr_write_final_pte()
1293 void hl_mmu_dr_clear_pte(struct hl_ctx *ctx, u64 pte_addr) in hl_mmu_dr_clear_pte() argument
1295 hl_mmu_dr_write_final_pte(ctx, pte_addr, 0); in hl_mmu_dr_clear_pte()
1298 void hl_mmu_dr_get_pte(struct hl_ctx *ctx, u64 hop_addr) in hl_mmu_dr_get_pte() argument
1300 hl_mmu_dr_get_pgt_info(ctx, hop_addr)->num_of_ptes++; in hl_mmu_dr_get_pte()
1303 int hl_mmu_dr_put_pte(struct hl_ctx *ctx, u64 hop_addr) in hl_mmu_dr_put_pte() argument
1305 struct pgt_info *pgt_info = hl_mmu_dr_get_pgt_info(ctx, hop_addr); in hl_mmu_dr_put_pte()
1308 pgt_info->num_of_ptes--; in hl_mmu_dr_put_pte()
1314 num_of_ptes_left = pgt_info->num_of_ptes; in hl_mmu_dr_put_pte()
1316 hl_mmu_dr_free_pgt_node(ctx, pgt_info); in hl_mmu_dr_put_pte()
1321 u64 hl_mmu_dr_alloc_hop(struct hl_ctx *ctx) in hl_mmu_dr_alloc_hop() argument
1323 struct hl_device *hdev = ctx->hdev; in hl_mmu_dr_alloc_hop()
1324 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_mmu_dr_alloc_hop()
1332 phys_addr = (u64) gen_pool_alloc(hdev->mmu_priv.dr.mmu_pgt_pool, in hl_mmu_dr_alloc_hop()
1333 prop->dmmu.hop_table_size); in hl_mmu_dr_alloc_hop()
1335 dev_err(hdev->dev, "failed to allocate page\n"); in hl_mmu_dr_alloc_hop()
1339 shadow_addr = (u64) (uintptr_t) kzalloc(prop->dmmu.hop_table_size, in hl_mmu_dr_alloc_hop()
1344 pgt_info->phys_addr = phys_addr; in hl_mmu_dr_alloc_hop()
1345 pgt_info->shadow_addr = shadow_addr; in hl_mmu_dr_alloc_hop()
1346 pgt_info->ctx = ctx; in hl_mmu_dr_alloc_hop()
1347 pgt_info->num_of_ptes = 0; in hl_mmu_dr_alloc_hop()
1348 hash_add(ctx->mmu_shadow_hash, &pgt_info->node, shadow_addr); in hl_mmu_dr_alloc_hop()
1353 gen_pool_free(hdev->mmu_priv.dr.mmu_pgt_pool, in hl_mmu_dr_alloc_hop()
1354 phys_addr, prop->dmmu.hop_table_size); in hl_mmu_dr_alloc_hop()
1361 u64 hl_mmu_dr_get_alloc_next_hop_addr(struct hl_ctx *ctx, u64 curr_pte, bool *is_new_hop) in hl_mmu_dr_get_alloc_next_hop_addr() argument
1363 u64 hop_addr = hl_mmu_get_next_hop_addr(ctx, curr_pte); in hl_mmu_dr_get_alloc_next_hop_addr()
1366 hop_addr = hl_mmu_dr_alloc_hop(ctx); in hl_mmu_dr_get_alloc_next_hop_addr()
1373 void hl_mmu_dr_flush(struct hl_ctx *ctx) in hl_mmu_dr_flush() argument
1377 ctx->hdev->asic_funcs->read_pte(ctx->hdev, hl_mmu_dr_get_phys_hop0_addr(ctx)); in hl_mmu_dr_flush()
1382 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_mmu_dr_init()
1385 hdev->mmu_priv.dr.mmu_pgt_pool = in hl_mmu_dr_init()
1386 gen_pool_create(__ffs(prop->dmmu.hop_table_size), -1); in hl_mmu_dr_init()
1388 if (!hdev->mmu_priv.dr.mmu_pgt_pool) { in hl_mmu_dr_init()
1389 dev_err(hdev->dev, "Failed to create page gen pool\n"); in hl_mmu_dr_init()
1390 return -ENOMEM; in hl_mmu_dr_init()
1393 rc = gen_pool_add(hdev->mmu_priv.dr.mmu_pgt_pool, prop->mmu_pgt_addr + in hl_mmu_dr_init()
1394 prop->dmmu.hop0_tables_total_size, in hl_mmu_dr_init()
1395 prop->dmmu.pgt_size - prop->dmmu.hop0_tables_total_size, in hl_mmu_dr_init()
1396 -1); in hl_mmu_dr_init()
1398 dev_err(hdev->dev, "Failed to add memory to page gen pool\n"); in hl_mmu_dr_init()
1402 hdev->mmu_priv.dr.mmu_shadow_hop0 = kvcalloc(prop->max_asid, in hl_mmu_dr_init()
1403 prop->dmmu.hop_table_size, GFP_KERNEL); in hl_mmu_dr_init()
1404 if (ZERO_OR_NULL_PTR(hdev->mmu_priv.dr.mmu_shadow_hop0)) { in hl_mmu_dr_init()
1405 rc = -ENOMEM; in hl_mmu_dr_init()
1414 gen_pool_destroy(hdev->mmu_priv.dr.mmu_pgt_pool); in hl_mmu_dr_init()
1423 if (ZERO_OR_NULL_PTR(hdev->mmu_priv.dr.mmu_shadow_hop0)) in hl_mmu_dr_fini()
1426 kvfree(hdev->mmu_priv.dr.mmu_shadow_hop0); in hl_mmu_dr_fini()
1427 gen_pool_destroy(hdev->mmu_priv.dr.mmu_pgt_pool); in hl_mmu_dr_fini()
1433 hdev->mmu_priv.dr.mmu_shadow_hop0 = NULL; in hl_mmu_dr_fini()