Lines Matching +full:compare +full:- +full:and +full:- +full:swap
1 /* SPDX-License-Identifier: GPL-2.0 */
9 * Derived from "include/asm-i386/pgtable.h"
17 #include <linux/page-flags.h>
18 #include <linux/radix-tree.h>
55 * for zero-mapped memory areas etc..
80 * The vmalloc and module area will always be on the topmost area of the
84 * happen without trampolines and in addition the placement within a
89 #define VMALLOC_DEFAULT_SIZE ((512UL << 30) - MODULES_LEN)
110 #define KMSAN_VMALLOC_SIZE (VMALLOC_END - VMALLOC_START)
135 * I Page-Invalid Bit: Page is not available for address-translation
136 * P Page-Protection Bit: Store access not possible for page
137 * C Change-bit override: HW is not required to set change bit
140 * | P-table origin | TT
144 * I Segment-Invalid Bit: Segment is not available for address-translation
145 * C Common-Segment Bit: Segment is not private (PoP 3-30)
146 * P Page-Protection Bit: Store access not possible for page
150 * | S-table origin | TF TTTL
154 * I Segment-Invalid Bit: Segment is not available for address-translation
164 * X Space-Switch event:
165 * G Segment-Invalid Bit:
166 * P Private-Space Bit:
167 * S Storage-Alteration:
169 * TL Table-Length:
181 #define _PAGE_NOEXEC 0x100 /* HW no-execute bit */
182 #define _PAGE_PROTECT 0x200 /* HW read-only bit */
203 #define _PAGE_SWP_EXCLUSIVE _PAGE_LARGE /* SW pte exclusive swap bit */
211 * HW bit and all SW bits.
216 * handle_pte_fault uses pte_present and pte_none to find out the pte type
218 * distinguish present from not-present ptes. It is changed only with the page
222 * the pte hardware and software bits in the last 12 bits of a pte
223 * (. unassigned bit, x don't care, t swap type):
230 * swap .11..ttttt.0
231 * prot-none, clean, old .11.xx0000.1
232 * prot-none, clean, young .11.xx0001.1
233 * prot-none, dirty, old .11.xx0010.1
234 * prot-none, dirty, young .11.xx0011.1
235 * read-only, clean, old .11.xx0100.1
236 * read-only, clean, young .01.xx0101.1
237 * read-only, dirty, old .11.xx0110.1
238 * read-only, dirty, young .01.xx0111.1
239 * read-write, clean, old .11.xx1100.1
240 * read-write, clean, young .01.xx1101.1
241 * read-write, dirty, old .10.xx1110.1
242 * read-write, dirty, young .00.xx1111.1
243 * HW-bits: R read-only, I invalid
244 * SW-bits: p present, y young, d dirty, r read, w write, s special,
252 /* Bits in the segment/region table address-space-control-element */
268 #define _REGION_ENTRY_NOEXEC 0x100 /* region no-execute bit */
290 #define _REGION3_ENTRY_COMM 0x0010 /* Common-Region, marks swap entry */
291 #define _REGION3_ENTRY_LARGE 0x0400 /* RTTE-format control, large page */
304 * SW region present bit. For non-leaf region-third-table entries, bits 62-63
305 * indicate the TABLE LENGTH and both must be set to 1. But such entries
318 #define _SEGMENT_ENTRY_NOEXEC 0x100 /* segment no-execute bit */
328 #define _SEGMENT_ENTRY_COMM 0x0010 /* Common-Segment, marks swap entry */
329 #define _SEGMENT_ENTRY_LARGE 0x0400 /* STE-format control, large page */
341 /* Common bits in region and segment table entries, for swap entries */
342 #define _RST_ENTRY_COMM 0x0010 /* Common-Region/Segment, marks swap entry */
367 #define _REGION1_MASK (~(_REGION1_SIZE - 1))
368 #define _REGION2_MASK (~(_REGION2_SIZE - 1))
369 #define _REGION3_MASK (~(_REGION3_SIZE - 1))
370 #define _SEGMENT_MASK (~(_SEGMENT_SIZE - 1))
394 * Segment table and region3 table entry encoding
395 * (R = read-only, I = invalid, y = young bit):
397 * prot-none, clean, old 00..1...1...00
398 * prot-none, clean, young 01..1...1...00
399 * prot-none, dirty, old 10..1...1...00
400 * prot-none, dirty, young 11..1...1...00
401 * read-only, clean, old 00..1...1...01
402 * read-only, clean, young 01..1...0...01
403 * read-only, dirty, old 10..1...1...01
404 * read-only, dirty, young 11..1...0...01
405 * read-write, clean, old 00..1...1...11
406 * read-write, clean, young 01..1...0...11
407 * read-write, dirty, old 10..0...1...11
408 * read-write, dirty, young 11..0...0...11
410 * read-write, old segment table entries (origin!=0)
411 * HW-bits: R read-only, I invalid
412 * SW-bits: y young, d dirty, r read, w write
438 * A user page table pointer has the space-switch-event bit, the
439 * private-space-control bit and the storage-alteration-event-control
552 return mm->context.asce_limit <= _REGION1_SIZE; in mm_p4d_folded()
558 return mm->context.asce_limit <= _REGION2_SIZE; in mm_pud_folded()
564 return mm->context.asce_limit <= _REGION3_SIZE; in mm_pmd_folded()
571 if (unlikely(mm->context.has_pgste)) in mm_has_pgste()
580 if (unlikely(atomic_read(&mm->context.protected_count))) in mm_is_protected()
589 if (unlikely(mm->context.alloc_pgste)) in mm_alloc_pgste()
627 * mapped shared zeropages and prevent new shared zeropages from getting
634 if (!mm->context.allow_cow_sharing) in mm_forbids_zeropage()
643 if (mm->context.uses_skeys) in mm_uses_skeys()
662 * cspg() - Compare and Swap and Purge (CSPG)
667 * Return: True if compare and swap was successful, otherwise false.
689 * crdte() - Compare and Replace DAT Table Entry
697 * Return: True if compare and replace was successful, otherwise false.
991 * usable for kernel address space mappings where fault driven dirty and
992 * young/old accounting is not supported, i.e _PAGE_PROTECT and _PAGE_INVALID
1075 * newprot for PAGE_NONE, PAGE_RO, PAGE_RX, PAGE_RW and PAGE_RWX in pte_modify()
1081 * newprot for PAGE_RO, PAGE_RX, PAGE_RW and PAGE_RWX has the page in pte_modify()
1155 pto = __pa(ptep) & ~(PTRS_PER_PTE * sizeof(pte_t) - 1); in __ptep_rdp()
1200 * This is hard to understand. ptep_get_and_clear and ptep_clear_flush
1221 pte = ptep_xchg_direct(vma->vm_mm, addr, ptep, pte_mkold(pte)); in ptep_test_and_clear_young()
1256 res = ptep_xchg_direct(vma->vm_mm, addr, ptep, __pte(_PAGE_INVALID)); in ptep_clear_flush()
1258 if (mm_is_protected(vma->vm_mm) && pte_present(res)) in ptep_clear_flush()
1268 * full==1 and a simple pte_clear is enough. See tlb.h.
1294 * If something went wrong and the page could not be destroyed, or in ptep_get_and_clear_full()
1314 * bits in the comparison. Those might change e.g. because of dirty and young
1338 * Therefore, only do the local TLB flush when RDP can be used, and the in flush_tlb_fix_spurious_fault()
1357 if (MACHINE_HAS_RDP && !mm_has_pgste(vma->vm_mm) && pte_allow_rdp(*ptep, entry)) in ptep_set_access_flags()
1358 ptep_reset_dat_prot(vma->vm_mm, addr, ptep, entry); in ptep_set_access_flags()
1360 ptep_xchg_direct(vma->vm_mm, addr, ptep, entry); in ptep_set_access_flags()
1409 * are within the same folio, PMD and VMA.
1419 if (--nr == 0) in set_ptes()
1428 if (--nr == 0) in set_ptes()
1438 * Conversion functions: convert a page and protection to a page entry,
1439 * and a page entry and page directory to the page they refer to.
1459 #define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))
1460 #define p4d_index(address) (((address) >> P4D_SHIFT) & (PTRS_PER_P4D-1))
1461 #define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1))
1462 #define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1))
1499 * The pgd_offset function *always* adds the index for the top-level
1502 * pgdp = pgd_offset(current->mm, addr);
1506 * The subsequent p4d_offset, pud_offset and pmd_offset functions
1518 return pgd + ((address >> shift) & (PTRS_PER_PGD - 1)); in pgd_offset_raw()
1521 #define pgd_offset(mm, address) pgd_offset_raw(READ_ONCE((mm)->pgd), address)
1571 return end <= current->mm->context.asce_limit; in gup_fast_permitted()
1716 sto = __pa(pmdp) - pmd_index(addr) * sizeof(pmd_t); in __pmdp_idte()
1742 r3o = __pa(pudp) - pud_index(addr) * sizeof(pud_t); in __pudp_idte()
1788 pmdp_xchg_direct(vma->vm_mm, addr, pmdp, entry); in pmdp_set_access_flags()
1798 pmd = pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd_mkold(pmd)); in pmdp_test_and_clear_young()
1840 return pmdp_xchg_lazy(vma->vm_mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY)); in pmdp_huge_get_and_clear_full()
1847 return pmdp_huge_get_and_clear(vma->vm_mm, addr, pmdp); in pmdp_huge_clear_flush()
1858 return pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd); in pmdp_invalidate()
1875 return pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp); in pmdp_collapse_flush()
1895 * 64 bit swap entry format:
1896 * A page-table entry has some bits we have to treat in a special way.
1897 * Bits 54 and 63 are used to indicate the page type. Bit 53 marks the pte
1899 * A swap pte is indicated by bit pattern (pte & 0x201) == 0x200
1904 * Bits 0-51 store the offset.
1906 * Bits 57-61 store the type.
1908 * Bits 55 and 56 (X) are unused.
1911 #define __SWP_OFFSET_MASK ((1UL << 52) - 1)
1913 #define __SWP_TYPE_MASK ((1UL << 5) - 1)
1945 * 64 bit swap entry format for REGION3 and SEGMENT table entries (RSTE)
1946 * Bits 59 and 63 are used to indicate the swap entry. Bit 58 marks the rste
1948 * A swap entry is indicated by bit pattern (rste & 0x011) == 0x010
1953 * Bits 0-51 store the offset.
1954 * Bits 53-57 store the type.
1956 * Bits 60-61 (TT) indicate the table type: 0x01 for REGION3 and 0x00 for SEGMENT.
1960 #define __SWP_OFFSET_MASK_RSTE ((1UL << 52) - 1)
1962 #define __SWP_TYPE_MASK_RSTE ((1UL << 5) - 1)
2006 ((pgtable_t)__va(pmd_val(pmd) & -sizeof(pte_t)*PTRS_PER_PTE))