1 // SPDX-License-Identifier: GPL-2.0-only
14 #define KVM_PGTABLE_FIRST_LEVEL		-1
18  * The largest supported block sizes for KVM (no 52-bit PA support):
19  *  - 4K (level 1):	1GB
20  *  - 16K (level 2):	32MB
21  *  - 64K (level 2):	512MB
60 #define KVM_PHYS_INVALID		(-1ULL)
99  * Used to indicate a pte for which a 'break-before-make' sequence is in
149 static inline u64 kvm_granule_shift(s8 level)  in kvm_granule_shift()  argument
152 	return ARM64_HW_PGTABLE_LEVEL_SHIFT(level);  in kvm_granule_shift()
155 static inline u64 kvm_granule_size(s8 level)  in kvm_granule_size()  argument
157 	return BIT(kvm_granule_shift(level));  in kvm_granule_size()
160 static inline bool kvm_level_supports_block_mapping(s8 level)  in kvm_level_supports_block_mapping()  argument
162 	return level >= KVM_PGTABLE_MIN_BLOCK_LEVEL;  in kvm_level_supports_block_mapping()
167 	s8 level = KVM_PGTABLE_MIN_BLOCK_LEVEL;  in kvm_supported_block_sizes()  local
170 	for (; level <= KVM_PGTABLE_LAST_LEVEL; level++)  in kvm_supported_block_sizes()
171 		r |= BIT(kvm_granule_shift(level));  in kvm_supported_block_sizes()
184  * struct kvm_pgtable_mm_ops - Memory management callbacks.
185  * @zalloc_page:		Allocate a single zeroed memory page.
188  *				the page is 1.
191  *				up to the next page boundary. The resulting
197  * @get_page:			Increment the refcount on a page.
198  * @put_page:			Decrement the refcount on a page. When the
199  *				refcount reaches 0 the page is automatically
201  * @page_count:			Return the refcount of a page.
215 	void		(*free_unlinked_table)(void *addr, s8 level);
226  * enum kvm_pgtable_stage2_flags - Stage-2 page-table flags.
227  * @KVM_PGTABLE_S2_NOFWB:	Don't enforce Normal-WB even if the CPUs have
237  * enum kvm_pgtable_prot - Page-table permissions and attributes.
277  * enum kvm_pgtable_walk_flags - Flags to control a depth-first page-table walk.
284  * @KVM_PGTABLE_WALK_SHARED:		Indicates the page-tables may be shared
286  * @KVM_PGTABLE_WALK_HANDLE_FAULT:	Indicates the page-table walk was
289  *					without Break-before-make's
313 	s8					level;  member
322 	return ctx->flags & KVM_PGTABLE_WALK_SHARED;  in kvm_pgtable_walk_shared()
326  * struct kvm_pgtable_walker - Hook into a page-table walk.
329  * @flags:	Bitwise-OR of flags to identify the entry types on which to
339  * RCU cannot be used in a non-kernel context such as the hyp. As such, page
357 	 * non-shared table walkers are allowed in the hypervisor.  in kvm_pgtable_walk_begin()
359 	if (walker->flags & KVM_PGTABLE_WALK_SHARED)  in kvm_pgtable_walk_begin()
360 		return -EPERM;  in kvm_pgtable_walk_begin()
379 	return rcu_dereference_check(pteref, !(walker->flags & KVM_PGTABLE_WALK_SHARED));  in kvm_dereference_pteref()
384 	if (walker->flags & KVM_PGTABLE_WALK_SHARED)  in kvm_pgtable_walk_begin()
392 	if (walker->flags & KVM_PGTABLE_WALK_SHARED)  in kvm_pgtable_walk_end()
404  * struct kvm_pgtable - KVM page-table.
406  * @start_level:	Level at which the page-table walk starts.
407  * @pgd:		Pointer to the first top-level entry of the page-table.
409  * @mmu:		Stage-2 KVM MMU struct. Unused for stage-1 page-tables.
410  * @flags:		Stage-2 page-table flags.
411  * @force_pte_cb:	Function that returns true if page level mappings must
423 			/* Stage-2 only */
432  * kvm_pgtable_hyp_init() - Initialise a hypervisor stage-1 page-table.
433  * @pgt:	Uninitialised page-table structure to initialise.
443  * kvm_pgtable_hyp_destroy() - Destroy an unused hypervisor stage-1 page-table.
444  * @pgt:	Page-table structure initialised by kvm_pgtable_hyp_init().
446  * The page-table is assumed to be unreachable by any hardware walkers prior
452  * kvm_pgtable_hyp_map() - Install a mapping in a hypervisor stage-1 page-table.
453  * @pgt:	Page-table structure initialised by kvm_pgtable_hyp_init().
459  * The offset of @addr within a page is ignored, @size is rounded-up to
460  * the next page boundary and @phys is rounded-down to the previous page
474  * kvm_pgtable_hyp_unmap() - Remove a mapping from a hypervisor stage-1 page-table.
475  * @pgt:	Page-table structure initialised by kvm_pgtable_hyp_init().
479  * The offset of @addr within a page is ignored, @size is rounded-up to
480  * the next page boundary and @phys is rounded-down to the previous page
483  * TLB invalidation is performed for each page-table entry cleared during the
484  * unmapping operation and the reference count for the page-table page
487  * invalid page-table entry or a valid block mapping which maps beyond the range
495  * kvm_get_vtcr() - Helper to construct VTCR_EL2
511  * kvm_pgtable_stage2_pgd_size() - Helper to compute size of a stage-2 PGD
514  * Return: the size (in bytes) of the stage-2 PGD
519  * __kvm_pgtable_stage2_init() - Initialise a guest stage-2 page-table.
520  * @pgt:	Uninitialised page-table structure to initialise.
523  * @flags:	Stage-2 configuration flags.
524  * @force_pte_cb: Function that returns true if page level mappings must
541  * kvm_pgtable_stage2_destroy() - Destroy an unused guest stage-2 page-table.
542  * @pgt:	Page-table structure initialised by kvm_pgtable_stage2_init*().
544  * The page-table is assumed to be unreachable by any hardware walkers prior
550  * kvm_pgtable_stage2_free_unlinked() - Free an unlinked stage-2 paging structure.
552  * @pgtable:	Unlinked stage-2 paging structure to be freed.
553  * @level:	Level of the stage-2 paging structure to be freed.
555  * The page-table is assumed to be unreachable by any hardware walkers prior to
558 void kvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops *mm_ops, void *pgtable, s8 level);
561  * kvm_pgtable_stage2_create_unlinked() - Create an unlinked stage-2 paging structure.
562  * @pgt:	Page-table structure initialised by kvm_pgtable_stage2_init*().
564  * @level:	Starting level of the stage-2 paging structure to be created.
566  * @mc:		Cache of pre-allocated and zeroed memory from which to allocate
567  *		page-table pages.
570  * Returns an unlinked page-table tree.  This new page-table tree is
572  * therefore unreachableby the hardware page-table walker. No TLB
578  * Return: The fully populated (unlinked) stage-2 paging structure, or
582 					      u64 phys, s8 level,
587  * kvm_pgtable_stage2_map() - Install a mapping in a guest stage-2 page-table.
588  * @pgt:	Page-table structure initialised by kvm_pgtable_stage2_init*().
593  * @mc:		Cache of pre-allocated and zeroed memory from which to allocate
594  *		page-table pages.
595  * @flags:	Flags to control the page-table walk (ex. a shared walk)
597  * The offset of @addr within a page is ignored, @size is rounded-up to
598  * the next page boundary and @phys is rounded-down to the previous page
610  * existing block mappings, relying on page-faults to fault back areas outside
620  * kvm_pgtable_stage2_set_owner() - Unmap and annotate pages in the IPA space to
622  * @pgt:	Page-table structure initialised by kvm_pgtable_stage2_init*().
625  * @mc:		Cache of pre-allocated and zeroed memory from which to allocate
626  *		page-table pages.
627  * @owner_id:	Unique identifier for the owner of the page.
629  * By default, all page-tables are owned by identifier 0. This function can be
631  * stage 2 is used with identity-mappings, these annotations allow to use the
632  * page-table data structure as a simple rmap.
640  * kvm_pgtable_stage2_unmap() - Remove a mapping from a guest stage-2 page-table.
641  * @pgt:	Page-table structure initialised by kvm_pgtable_stage2_init*().
645  * The offset of @addr within a page is ignored and @size is rounded-up to
646  * the next page boundary.
648  * TLB invalidation is performed for each page-table entry cleared during the
649  * unmapping operation and the reference count for the page-table page
651  * freed. Unmapping a cacheable page will ensure that it is clean to the PoC if
659  * kvm_pgtable_stage2_wrprotect() - Write-protect guest stage-2 address range
661  * @pgt:	Page-table structure initialised by kvm_pgtable_stage2_init*().
662  * @addr:	Intermediate physical address from which to write-protect,
665  * The offset of @addr within a page is ignored and @size is rounded-up to
666  * the next page boundary.
677  * kvm_pgtable_stage2_mkyoung() - Set the access flag in a page-table entry.
678  * @pgt:	Page-table structure initialised by kvm_pgtable_stage2_init*().
679  * @addr:	Intermediate physical address to identify the page-table entry.
680  * @flags:	Flags to control the page-table walk (ex. a shared walk)
682  * The offset of @addr within a page is ignored.
684  * If there is a valid, leaf page-table entry used to translate @addr, then
691  * kvm_pgtable_stage2_test_clear_young() - Test and optionally clear the access
692  *					   flag in a page-table entry.
693  * @pgt:	Page-table structure initialised by kvm_pgtable_stage2_init*().
694  * @addr:	Intermediate physical address to identify the page-table entry.
698  * The offset of @addr within a page is ignored.
701  * page-table entry used to translate the range [@addr, @addr + @size).
713  * kvm_pgtable_stage2_relax_perms() - Relax the permissions enforced by a
714  *				      page-table entry.
715  * @pgt:	Page-table structure initialised by kvm_pgtable_stage2_init*().
716  * @addr:	Intermediate physical address to identify the page-table entry.
718  * @flags:	Flags to control the page-table walk (ex. a shared walk)
720  * The offset of @addr within a page is ignored.
722  * If there is a valid, leaf page-table entry used to translate @addr, then
735  * kvm_pgtable_stage2_flush_range() - Clean and invalidate data cache to Point
736  * 				      of Coherency for guest stage-2 address
738  * @pgt:	Page-table structure initialised by kvm_pgtable_stage2_init*().
742  * The offset of @addr within a page is ignored and @size is rounded-up to
743  * the next page boundary.
750  * kvm_pgtable_stage2_split() - Split a range of huge pages into leaf PTEs pointing
752  * @pgt:	 Page-table structure initialised by kvm_pgtable_stage2_init().
755  * @mc:		 Cache of pre-allocated and zeroed memory from which to allocate
756  *		 page-table pages.
758  * The function tries to split any level 1 or 2 entry that overlaps
769  * kvm_pgtable_walk() - Walk a page-table.
770  * @pgt:	Page-table structure initialised by kvm_pgtable_*_init().
775  * The offset of @addr within a page is ignored and @size is rounded-up to
776  * the next page boundary.
778  * The walker will walk the page-table entries corresponding to the input
780  * Invalid entries are treated as leaf entries. The visited page table entry is
793  * kvm_pgtable_get_leaf() - Walk a page-table and retrieve the leaf entry
794  *			    with its level.
795  * @pgt:	Page-table structure initialised by kvm_pgtable_*_init()
799  * @level:	Pointer to storage for the level of the retrieved PTE.
801  * The offset of @addr within a page is ignored.
803  * The walker will walk the page-table entries corresponding to the input
810 			 kvm_pte_t *ptep, s8 *level);
813  * kvm_pgtable_stage2_pte_prot() - Retrieve the protection attributes of a
814  *				   stage-2 Page-Table Entry.
815  * @pte:	Page-table entry
817  * Return: protection attributes of the page-table entry in the enum
823  * kvm_pgtable_hyp_pte_prot() - Retrieve the protection attributes of a stage-1
824  *				Page-Table Entry.
825  * @pte:	Page-table entry
827  * Return: protection attributes of the page-table entry in the enum
833  * kvm_tlb_flush_vmid_range() - Invalidate/flush a range of TLB entries
835  * @mmu:	Stage-2 KVM MMU struct