1 // SPDX-License-Identifier: GPL-2.0
2 /* arch/sparc64/mm/tlb.c
9 #include <linux/mm.h>
26 	struct mm_struct *mm = tb->mm;  in flush_tlb_pending()  local
28 	if (!tb->tlb_nr)  in flush_tlb_pending()
33 	if (CTX_VALID(mm->context)) {  in flush_tlb_pending()
34 		if (tb->tlb_nr == 1) {  in flush_tlb_pending()
35 			global_flush_tlb_page(mm, tb->vaddrs[0]);  in flush_tlb_pending()
38 			smp_flush_tlb_pending(tb->mm, tb->tlb_nr,  in flush_tlb_pending()
39 					      &tb->vaddrs[0]);  in flush_tlb_pending()
41 			__flush_tlb_pending(CTX_HWBITS(tb->mm->context),  in flush_tlb_pending()
42 					    tb->tlb_nr, &tb->vaddrs[0]);  in flush_tlb_pending()
47 	tb->tlb_nr = 0;  in flush_tlb_pending()
59 	tb->active = 1;  in arch_enter_lazy_mmu_mode()
66 	if (tb->tlb_nr)  in arch_leave_lazy_mmu_mode()
68 	tb->active = 0;  in arch_leave_lazy_mmu_mode()
72 static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr,  in tlb_batch_add_one()  argument
80 		vaddr |= 0x1UL;  in tlb_batch_add_one()
82 	nr = tb->tlb_nr;  in tlb_batch_add_one()
84 	if (unlikely(nr != 0 && mm != tb->mm)) {  in tlb_batch_add_one()
86 		nr = 0;  in tlb_batch_add_one()
89 	if (!tb->active) {  in tlb_batch_add_one()
90 		flush_tsb_user_page(mm, vaddr, hugepage_shift);  in tlb_batch_add_one()
91 		global_flush_tlb_page(mm, vaddr);  in tlb_batch_add_one()
95 	if (nr == 0) {  in tlb_batch_add_one()
96 		tb->mm = mm;  in tlb_batch_add_one()
97 		tb->hugepage_shift = hugepage_shift;  in tlb_batch_add_one()
100 	if (tb->hugepage_shift != hugepage_shift) {  in tlb_batch_add_one()
102 		tb->hugepage_shift = hugepage_shift;  in tlb_batch_add_one()
103 		nr = 0;  in tlb_batch_add_one()
106 	tb->vaddrs[nr] = vaddr;  in tlb_batch_add_one()
107 	tb->tlb_nr = ++nr;  in tlb_batch_add_one()
115 void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,  in tlb_batch_add()  argument
141 			flush_dcache_folio_all(mm, folio);  in tlb_batch_add()
146 		tlb_batch_add_one(mm, vaddr, pte_exec(orig), hugepage_shift);  in tlb_batch_add()
150 static void tlb_batch_pmd_scan(struct mm_struct *mm, unsigned long vaddr,  in tlb_batch_pmd_scan()  argument
164 			tlb_batch_add_one(mm, vaddr, exec, PAGE_SHIFT);  in tlb_batch_pmd_scan()
173 static void __set_pmd_acct(struct mm_struct *mm, unsigned long addr,  in __set_pmd_acct()  argument
176 	if (mm == &init_mm)  in __set_pmd_acct()
190 				mm->context.hugetlb_pte_count++;  in __set_pmd_acct()
192 				mm->context.thp_pte_count++;  in __set_pmd_acct()
195 				mm->context.hugetlb_pte_count--;  in __set_pmd_acct()
197 				mm->context.thp_pte_count--;  in __set_pmd_acct()
216 			tlb_batch_add_one(mm, addr, exec, REAL_HPAGE_SHIFT);  in __set_pmd_acct()
217 			tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec,  in __set_pmd_acct()
220 			tlb_batch_pmd_scan(mm, addr, orig);  in __set_pmd_acct()
225 void set_pmd_at(struct mm_struct *mm, unsigned long addr,  in set_pmd_at()  argument
231 	__set_pmd_acct(mm, addr, orig, pmd);  in set_pmd_at()
241 	} while (cmpxchg64(&pmdp->pmd, old.pmd, pmd.pmd) != old.pmd);  in pmdp_establish()
242 	__set_pmd_acct(vma->vm_mm, address, old, pmd);  in pmdp_establish()
267 		(vma->vm_mm)->context.thp_pte_count--;  in pmdp_invalidate()
272 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,  in pgtable_trans_huge_deposit()  argument
277 	assert_spin_locked(&mm->page_table_lock);  in pgtable_trans_huge_deposit()
280 	if (!pmd_huge_pte(mm, pmdp))  in pgtable_trans_huge_deposit()
283 		list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));  in pgtable_trans_huge_deposit()
284 	pmd_huge_pte(mm, pmdp) = pgtable;  in pgtable_trans_huge_deposit()
287 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)  in pgtable_trans_huge_withdraw()  argument
292 	assert_spin_locked(&mm->page_table_lock);  in pgtable_trans_huge_withdraw()
295 	pgtable = pmd_huge_pte(mm, pmdp);  in pgtable_trans_huge_withdraw()
298 		pmd_huge_pte(mm, pmdp) = NULL;  in pgtable_trans_huge_withdraw()
300 		pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;  in pgtable_trans_huge_withdraw()
303 	pte_val(pgtable[0]) = 0;  in pgtable_trans_huge_withdraw()
304 	pte_val(pgtable[1]) = 0;  in pgtable_trans_huge_withdraw()