1 // SPDX-License-Identifier: GPL-2.0
4  * Copyright 2016-2020 HabanaLabs, Ltd.
15  * hl_mmu_v2_ctx_init() - initialize a context for using the MMU module.
16  * @ctx: pointer to the context structure to initialize.
20  * Return: 0 on success, non-zero otherwise.
22 static int hl_mmu_v2_ctx_init(struct hl_ctx *ctx)  in hl_mmu_v2_ctx_init()  argument
24 	hash_init(ctx->mmu_shadow_hash);  in hl_mmu_v2_ctx_init()
30  * hl_mmu_v2_ctx_fini - disable a ctx from using the mmu module
32  * @ctx: pointer to the context structure
35  * - Free any pgts which were not freed yet
36  * - Free the mutex
37  * - Free DRAM default page mapping hops
39 static void hl_mmu_v2_ctx_fini(struct hl_ctx *ctx)  in hl_mmu_v2_ctx_fini()  argument
41 	struct hl_device *hdev = ctx->hdev;  in hl_mmu_v2_ctx_fini()
46 	if (!hash_empty(ctx->mmu_shadow_hash))  in hl_mmu_v2_ctx_fini()
47 		dev_err(hdev->dev, "ctx %d is freed while it has pgts in use\n",  in hl_mmu_v2_ctx_fini()
48 			ctx->asid);  in hl_mmu_v2_ctx_fini()
50 	hash_for_each_safe(ctx->mmu_shadow_hash, i, tmp, pgt_info, node) {  in hl_mmu_v2_ctx_fini()
51 		dev_err_ratelimited(hdev->dev,  in hl_mmu_v2_ctx_fini()
52 			"pgt_info of addr 0x%llx of asid %d was not destroyed, num_ptes: %d\n",  in hl_mmu_v2_ctx_fini()
53 			pgt_info->phys_addr, ctx->asid, pgt_info->num_of_ptes);  in hl_mmu_v2_ctx_fini()
54 		hl_mmu_dr_free_pgt_node(ctx, pgt_info);  in hl_mmu_v2_ctx_fini()
58 static int hl_mmu_v2_unmap(struct hl_ctx *ctx,	u64 virt_addr, bool is_dram_addr)  in hl_mmu_v2_unmap()  argument
62 	struct asic_fixed_properties *prop = &ctx->hdev->asic_prop;  in hl_mmu_v2_unmap()
63 	struct hl_device *hdev = ctx->hdev;  in hl_mmu_v2_unmap()
70 		return -EINVAL;  in hl_mmu_v2_unmap()
72 	mmu_prop = &prop->dmmu;  in hl_mmu_v2_unmap()
74 	hop_last = mmu_prop->num_hops - 1;  in hl_mmu_v2_unmap()
76 	scrambled_virt_addr = hdev->asic_funcs->scramble_addr(hdev, virt_addr);  in hl_mmu_v2_unmap()
78 	hop_addr[0] = hl_mmu_dr_get_hop0_addr(ctx);  in hl_mmu_v2_unmap()
79 	hop_pte_addr[0] = hl_mmu_get_hop_pte_phys_addr(ctx, mmu_prop, 0,  in hl_mmu_v2_unmap()
82 		return -EFAULT;  in hl_mmu_v2_unmap()
86 	for (i = 1 ; i < mmu_prop->num_hops ; i++) {  in hl_mmu_v2_unmap()
87 		hop_addr[i] = hl_mmu_get_next_hop_addr(ctx, curr_pte);  in hl_mmu_v2_unmap()
91 		hop_pte_addr[i] = hl_mmu_get_hop_pte_phys_addr(ctx, mmu_prop, i,  in hl_mmu_v2_unmap()
94 			return -EFAULT;  in hl_mmu_v2_unmap()
98 		if ((i <= hop_last) && (curr_pte & mmu_prop->last_mask)) {  in hl_mmu_v2_unmap()
106 		dev_err(hdev->dev, "DRAM unmapping should use huge pages only\n");  in hl_mmu_v2_unmap()
107 		return -EFAULT;  in hl_mmu_v2_unmap()
113 	for (i = hop_last ; i > 0 ; i--) {  in hl_mmu_v2_unmap()
114 		hl_mmu_dr_clear_pte(ctx, hop_pte_addr[i]);  in hl_mmu_v2_unmap()
115 		if (hl_mmu_dr_put_pte(ctx, hop_addr[i]))  in hl_mmu_v2_unmap()
118 	hl_mmu_dr_clear_pte(ctx, hop_pte_addr[0]);  in hl_mmu_v2_unmap()
124 	dev_err(hdev->dev, "virt addr 0x%llx is not mapped to phys addr\n",  in hl_mmu_v2_unmap()
127 	return -EINVAL;  in hl_mmu_v2_unmap()
130 static int hl_mmu_v2_map(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr,  in hl_mmu_v2_map()  argument
135 	struct asic_fixed_properties *prop = &ctx->hdev->asic_prop;  in hl_mmu_v2_map()
137 	struct hl_device *hdev = ctx->hdev;  in hl_mmu_v2_map()
143 		return -EINVAL;  in hl_mmu_v2_map()
145 	mmu_prop = &prop->dmmu;  in hl_mmu_v2_map()
147 	hop_last = mmu_prop->num_hops - 1;  in hl_mmu_v2_map()
149 	scrambled_virt_addr = hdev->asic_funcs->scramble_addr(hdev, virt_addr);  in hl_mmu_v2_map()
150 	scrambled_phys_addr = hdev->asic_funcs->scramble_addr(hdev, phys_addr);  in hl_mmu_v2_map()
153 	hop_addr[0] = hl_mmu_dr_get_hop0_addr(ctx);  in hl_mmu_v2_map()
154 	hop_pte_addr[0] = hl_mmu_get_hop_pte_phys_addr(ctx, mmu_prop, 0,  in hl_mmu_v2_map()
160 		hop_addr[i] = hl_mmu_dr_get_alloc_next_hop_addr(ctx, curr_pte, &hop_new[i]);  in hl_mmu_v2_map()
162 			rc = -ENOMEM;  in hl_mmu_v2_map()
166 		hop_pte_addr[i] = hl_mmu_get_hop_pte_phys_addr(ctx, mmu_prop, i,  in hl_mmu_v2_map()
169 			rc = -EINVAL;  in hl_mmu_v2_map()
174 			rc = -EINVAL;  in hl_mmu_v2_map()
182 		dev_err(hdev->dev,  in hl_mmu_v2_map()
187 			dev_dbg(hdev->dev, "hop%d pte: 0x%llx (0x%llx)\n",  in hl_mmu_v2_map()
191 		rc = -EINVAL;  in hl_mmu_v2_map()
196 					| mmu_prop->last_mask | PAGE_PRESENT_MASK;  in hl_mmu_v2_map()
199 	hl_mmu_dr_write_final_pte(ctx, hop_pte_addr[hop_last], curr_pte);  in hl_mmu_v2_map()
201 	/* for each new hop, add its address to the table of previous-hop */  in hl_mmu_v2_map()
205 			hl_mmu_dr_write_pte(ctx, hop_pte_addr[i - 1], curr_pte);  in hl_mmu_v2_map()
207 			if (i - 1)  in hl_mmu_v2_map()
208 				hl_mmu_dr_get_pte(ctx, hop_addr[i - 1]);  in hl_mmu_v2_map()
211 	hl_mmu_dr_get_pte(ctx, hop_addr[hop_last]);  in hl_mmu_v2_map()
218 			hl_mmu_dr_free_hop(ctx, hop_addr[i]);  in hl_mmu_v2_map()
224  * hl_mmu_v2_swap_out - marks all mapping of the given ctx as swapped out
226  * @ctx: pointer to the context structure
229 static void hl_mmu_v2_swap_out(struct hl_ctx *ctx)  in hl_mmu_v2_swap_out()  argument
235  * hl_mmu_v2_swap_in - marks all mapping of the given ctx as swapped in
237  * @ctx: pointer to the context structure
240 static void hl_mmu_v2_swap_in(struct hl_ctx *ctx)  in hl_mmu_v2_swap_in()  argument
245 static int hl_mmu_v2_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr, struct hl_mmu_hop_info *hops)  in hl_mmu_v2_get_tlb_info()  argument
247 	struct asic_fixed_properties *prop = &ctx->hdev->asic_prop;  in hl_mmu_v2_get_tlb_info()
248 	struct hl_device *hdev = ctx->hdev;  in hl_mmu_v2_get_tlb_info()
253 	is_dram_addr = hl_mem_area_inside_range(virt_addr, prop->dmmu.page_size,  in hl_mmu_v2_get_tlb_info()
254 						prop->dmmu.start_addr,  in hl_mmu_v2_get_tlb_info()
255 						prop->dmmu.end_addr);  in hl_mmu_v2_get_tlb_info()
259 		return -EINVAL;  in hl_mmu_v2_get_tlb_info()
261 	mmu_prop = &prop->dmmu;  in hl_mmu_v2_get_tlb_info()
262 	hops->range_type = HL_VA_RANGE_TYPE_DRAM;  in hl_mmu_v2_get_tlb_info()
264 	hops->scrambled_vaddr = hdev->asic_funcs->scramble_addr(hdev, virt_addr);  in hl_mmu_v2_get_tlb_info()
266 	hops->hop_info[0].hop_addr = hl_mmu_dr_get_phys_hop0_addr(ctx);  in hl_mmu_v2_get_tlb_info()
267 	hops->hop_info[0].hop_pte_addr = hl_mmu_get_hop_pte_phys_addr(ctx, mmu_prop, 0,  in hl_mmu_v2_get_tlb_info()
268 						hops->hop_info[0].hop_addr,  in hl_mmu_v2_get_tlb_info()
269 							hops->scrambled_vaddr);  in hl_mmu_v2_get_tlb_info()
270 	if (hops->hop_info[0].hop_pte_addr == U64_MAX)  in hl_mmu_v2_get_tlb_info()
271 		return -EFAULT;  in hl_mmu_v2_get_tlb_info()
273 	hops->hop_info[0].hop_pte_val = hdev->asic_funcs->read_pte(hdev,  in hl_mmu_v2_get_tlb_info()
274 						hops->hop_info[0].hop_pte_addr);  in hl_mmu_v2_get_tlb_info()
275 	if (hops->hop_info[0].hop_pte_val == U64_MAX)  in hl_mmu_v2_get_tlb_info()
276 		return -EFAULT;  in hl_mmu_v2_get_tlb_info()
278 	for (i = 1 ; i < mmu_prop->num_hops ; i++) {  in hl_mmu_v2_get_tlb_info()
279 		hops->hop_info[i].hop_addr =  in hl_mmu_v2_get_tlb_info()
280 			hl_mmu_get_next_hop_addr(ctx, hops->hop_info[i - 1].hop_pte_val);  in hl_mmu_v2_get_tlb_info()
281 		if (hops->hop_info[i].hop_addr == ULLONG_MAX)  in hl_mmu_v2_get_tlb_info()
282 			return -EFAULT;  in hl_mmu_v2_get_tlb_info()
284 		hops->hop_info[i].hop_pte_addr =  in hl_mmu_v2_get_tlb_info()
285 				hl_mmu_get_hop_pte_phys_addr(ctx, mmu_prop, i,  in hl_mmu_v2_get_tlb_info()
286 						hops->hop_info[i].hop_addr,  in hl_mmu_v2_get_tlb_info()
287 						hops->scrambled_vaddr);  in hl_mmu_v2_get_tlb_info()
288 		if (hops->hop_info[i].hop_pte_addr == U64_MAX)  in hl_mmu_v2_get_tlb_info()
289 			return -EFAULT;  in hl_mmu_v2_get_tlb_info()
291 		hops->hop_info[i].hop_pte_val =  in hl_mmu_v2_get_tlb_info()
292 				hdev->asic_funcs->read_pte(hdev,  in hl_mmu_v2_get_tlb_info()
293 					hops->hop_info[i].hop_pte_addr);  in hl_mmu_v2_get_tlb_info()
295 		if (!(hops->hop_info[i].hop_pte_val & PAGE_PRESENT_MASK))  in hl_mmu_v2_get_tlb_info()
296 			return -EFAULT;  in hl_mmu_v2_get_tlb_info()
298 		if (hops->hop_info[i].hop_pte_val & mmu_prop->last_mask)  in hl_mmu_v2_get_tlb_info()
303 	if (i == mmu_prop->num_hops)  in hl_mmu_v2_get_tlb_info()
304 		return -EFAULT;  in hl_mmu_v2_get_tlb_info()
306 	if (!(hops->hop_info[i].hop_pte_val & PAGE_PRESENT_MASK))  in hl_mmu_v2_get_tlb_info()
307 		return -EFAULT;  in hl_mmu_v2_get_tlb_info()
309 	if (hops->scrambled_vaddr != virt_addr)  in hl_mmu_v2_get_tlb_info()
310 		hops->unscrambled_paddr = hdev->asic_funcs->descramble_addr  in hl_mmu_v2_get_tlb_info()
311 				(hdev, hops->hop_info[i].hop_pte_val);  in hl_mmu_v2_get_tlb_info()
313 		hops->unscrambled_paddr = hops->hop_info[i].hop_pte_val;  in hl_mmu_v2_get_tlb_info()
315 	hops->used_hops = i + 1;  in hl_mmu_v2_get_tlb_info()
321  * hl_mmu_v2_prepare - prepare mmu_if for working with mmu v2
328 	mmu->init = hl_mmu_dr_init;  in hl_mmu_v2_set_funcs()
329 	mmu->fini = hl_mmu_dr_fini;  in hl_mmu_v2_set_funcs()
330 	mmu->ctx_init = hl_mmu_v2_ctx_init;  in hl_mmu_v2_set_funcs()
331 	mmu->ctx_fini = hl_mmu_v2_ctx_fini;  in hl_mmu_v2_set_funcs()
332 	mmu->map = hl_mmu_v2_map;  in hl_mmu_v2_set_funcs()
333 	mmu->unmap = hl_mmu_v2_unmap;  in hl_mmu_v2_set_funcs()
334 	mmu->flush = hl_mmu_dr_flush;  in hl_mmu_v2_set_funcs()
335 	mmu->swap_out = hl_mmu_v2_swap_out;  in hl_mmu_v2_set_funcs()
336 	mmu->swap_in = hl_mmu_v2_swap_in;  in hl_mmu_v2_set_funcs()
337 	mmu->get_tlb_info = hl_mmu_v2_get_tlb_info;  in hl_mmu_v2_set_funcs()