1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef __ARM64_KVM_NESTED_H
3 #define __ARM64_KVM_NESTED_H
4 
5 #include <linux/bitfield.h>
6 #include <linux/kvm_host.h>
7 #include <asm/kvm_emulate.h>
8 #include <asm/kvm_pgtable.h>
9 
vcpu_has_nv(const struct kvm_vcpu * vcpu)10 static inline bool vcpu_has_nv(const struct kvm_vcpu *vcpu)
11 {
12 	return (!__is_defined(__KVM_NVHE_HYPERVISOR__) &&
13 		cpus_have_final_cap(ARM64_HAS_NESTED_VIRT) &&
14 		vcpu_has_feature(vcpu, KVM_ARM_VCPU_HAS_EL2));
15 }
16 
17 /* Translation helpers from non-VHE EL2 to EL1 */
tcr_el2_ps_to_tcr_el1_ips(u64 tcr_el2)18 static inline u64 tcr_el2_ps_to_tcr_el1_ips(u64 tcr_el2)
19 {
20 	return (u64)FIELD_GET(TCR_EL2_PS_MASK, tcr_el2) << TCR_IPS_SHIFT;
21 }
22 
translate_tcr_el2_to_tcr_el1(u64 tcr)23 static inline u64 translate_tcr_el2_to_tcr_el1(u64 tcr)
24 {
25 	return TCR_EPD1_MASK |				/* disable TTBR1_EL1 */
26 	       ((tcr & TCR_EL2_TBI) ? TCR_TBI0 : 0) |
27 	       tcr_el2_ps_to_tcr_el1_ips(tcr) |
28 	       (tcr & TCR_EL2_TG0_MASK) |
29 	       (tcr & TCR_EL2_ORGN0_MASK) |
30 	       (tcr & TCR_EL2_IRGN0_MASK) |
31 	       (tcr & TCR_EL2_T0SZ_MASK);
32 }
33 
translate_cptr_el2_to_cpacr_el1(u64 cptr_el2)34 static inline u64 translate_cptr_el2_to_cpacr_el1(u64 cptr_el2)
35 {
36 	u64 cpacr_el1 = CPACR_EL1_RES1;
37 
38 	if (cptr_el2 & CPTR_EL2_TTA)
39 		cpacr_el1 |= CPACR_EL1_TTA;
40 	if (!(cptr_el2 & CPTR_EL2_TFP))
41 		cpacr_el1 |= CPACR_EL1_FPEN;
42 	if (!(cptr_el2 & CPTR_EL2_TZ))
43 		cpacr_el1 |= CPACR_EL1_ZEN;
44 
45 	cpacr_el1 |= cptr_el2 & (CPTR_EL2_TCPAC | CPTR_EL2_TAM);
46 
47 	return cpacr_el1;
48 }
49 
translate_sctlr_el2_to_sctlr_el1(u64 val)50 static inline u64 translate_sctlr_el2_to_sctlr_el1(u64 val)
51 {
52 	/* Only preserve the minimal set of bits we support */
53 	val &= (SCTLR_ELx_M | SCTLR_ELx_A | SCTLR_ELx_C | SCTLR_ELx_SA |
54 		SCTLR_ELx_I | SCTLR_ELx_IESB | SCTLR_ELx_WXN | SCTLR_ELx_EE);
55 	val |= SCTLR_EL1_RES1;
56 
57 	return val;
58 }
59 
translate_ttbr0_el2_to_ttbr0_el1(u64 ttbr0)60 static inline u64 translate_ttbr0_el2_to_ttbr0_el1(u64 ttbr0)
61 {
62 	/* Clear the ASID field */
63 	return ttbr0 & ~GENMASK_ULL(63, 48);
64 }
65 
66 extern bool forward_smc_trap(struct kvm_vcpu *vcpu);
67 extern bool forward_debug_exception(struct kvm_vcpu *vcpu);
68 extern void kvm_init_nested(struct kvm *kvm);
69 extern int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu);
70 extern void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu);
71 extern struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu);
72 
73 union tlbi_info;
74 
75 extern void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid,
76 				       const union tlbi_info *info,
77 				       void (*)(struct kvm_s2_mmu *,
78 						const union tlbi_info *));
79 extern void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu);
80 extern void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu);
81 
82 extern void check_nested_vcpu_requests(struct kvm_vcpu *vcpu);
83 
84 struct kvm_s2_trans {
85 	phys_addr_t output;
86 	unsigned long block_size;
87 	bool writable;
88 	bool readable;
89 	int level;
90 	u32 esr;
91 	u64 desc;
92 };
93 
kvm_s2_trans_output(struct kvm_s2_trans * trans)94 static inline phys_addr_t kvm_s2_trans_output(struct kvm_s2_trans *trans)
95 {
96 	return trans->output;
97 }
98 
kvm_s2_trans_size(struct kvm_s2_trans * trans)99 static inline unsigned long kvm_s2_trans_size(struct kvm_s2_trans *trans)
100 {
101 	return trans->block_size;
102 }
103 
kvm_s2_trans_esr(struct kvm_s2_trans * trans)104 static inline u32 kvm_s2_trans_esr(struct kvm_s2_trans *trans)
105 {
106 	return trans->esr;
107 }
108 
kvm_s2_trans_readable(struct kvm_s2_trans * trans)109 static inline bool kvm_s2_trans_readable(struct kvm_s2_trans *trans)
110 {
111 	return trans->readable;
112 }
113 
kvm_s2_trans_writable(struct kvm_s2_trans * trans)114 static inline bool kvm_s2_trans_writable(struct kvm_s2_trans *trans)
115 {
116 	return trans->writable;
117 }
118 
kvm_s2_trans_executable(struct kvm_s2_trans * trans)119 static inline bool kvm_s2_trans_executable(struct kvm_s2_trans *trans)
120 {
121 	return !(trans->desc & BIT(54));
122 }
123 
124 extern int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,
125 			      struct kvm_s2_trans *result);
126 extern int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu,
127 				    struct kvm_s2_trans *trans);
128 extern int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2);
129 extern void kvm_nested_s2_wp(struct kvm *kvm);
130 extern void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block);
131 extern void kvm_nested_s2_flush(struct kvm *kvm);
132 
133 unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val);
134 
kvm_supported_tlbi_s1e1_op(struct kvm_vcpu * vpcu,u32 instr)135 static inline bool kvm_supported_tlbi_s1e1_op(struct kvm_vcpu *vpcu, u32 instr)
136 {
137 	struct kvm *kvm = vpcu->kvm;
138 	u8 CRm = sys_reg_CRm(instr);
139 
140 	if (!(sys_reg_Op0(instr) == TLBI_Op0 &&
141 	      sys_reg_Op1(instr) == TLBI_Op1_EL1))
142 		return false;
143 
144 	if (!(sys_reg_CRn(instr) == TLBI_CRn_XS ||
145 	      (sys_reg_CRn(instr) == TLBI_CRn_nXS &&
146 	       kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP))))
147 		return false;
148 
149 	if (CRm == TLBI_CRm_nROS &&
150 	    !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS))
151 		return false;
152 
153 	if ((CRm == TLBI_CRm_RIS || CRm == TLBI_CRm_ROS ||
154 	     CRm == TLBI_CRm_RNS) &&
155 	    !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE))
156 		return false;
157 
158 	return true;
159 }
160 
kvm_supported_tlbi_s1e2_op(struct kvm_vcpu * vpcu,u32 instr)161 static inline bool kvm_supported_tlbi_s1e2_op(struct kvm_vcpu *vpcu, u32 instr)
162 {
163 	struct kvm *kvm = vpcu->kvm;
164 	u8 CRm = sys_reg_CRm(instr);
165 
166 	if (!(sys_reg_Op0(instr) == TLBI_Op0 &&
167 	      sys_reg_Op1(instr) == TLBI_Op1_EL2))
168 		return false;
169 
170 	if (!(sys_reg_CRn(instr) == TLBI_CRn_XS ||
171 	      (sys_reg_CRn(instr) == TLBI_CRn_nXS &&
172 	       kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP))))
173 		return false;
174 
175 	if (CRm == TLBI_CRm_IPAIS || CRm == TLBI_CRm_IPAONS)
176 		return false;
177 
178 	if (CRm == TLBI_CRm_nROS &&
179 	    !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS))
180 		return false;
181 
182 	if ((CRm == TLBI_CRm_RIS || CRm == TLBI_CRm_ROS ||
183 	     CRm == TLBI_CRm_RNS) &&
184 	    !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE))
185 		return false;
186 
187 	return true;
188 }
189 
190 int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu);
191 
192 #ifdef CONFIG_ARM64_PTR_AUTH
193 bool kvm_auth_eretax(struct kvm_vcpu *vcpu, u64 *elr);
194 #else
kvm_auth_eretax(struct kvm_vcpu * vcpu,u64 * elr)195 static inline bool kvm_auth_eretax(struct kvm_vcpu *vcpu, u64 *elr)
196 {
197 	/* We really should never execute this... */
198 	WARN_ON_ONCE(1);
199 	*elr = 0xbad9acc0debadbad;
200 	return false;
201 }
202 #endif
203 
204 #define KVM_NV_GUEST_MAP_SZ	(KVM_PGTABLE_PROT_SW1 | KVM_PGTABLE_PROT_SW0)
205 
kvm_encode_nested_level(struct kvm_s2_trans * trans)206 static inline u64 kvm_encode_nested_level(struct kvm_s2_trans *trans)
207 {
208 	return FIELD_PREP(KVM_NV_GUEST_MAP_SZ, trans->level);
209 }
210 
211 /* Adjust alignment for the contiguous bit as per StageOA() */
212 #define contiguous_bit_shift(d, wi, l)					\
213 	({								\
214 		u8 shift = 0;						\
215 									\
216 		if ((d) & PTE_CONT) {					\
217 			switch (BIT((wi)->pgshift)) {			\
218 			case SZ_4K:					\
219 				shift = 4;				\
220 				break;					\
221 			case SZ_16K:					\
222 				shift = (l) == 2 ? 5 : 7;		\
223 				break;					\
224 			case SZ_64K:					\
225 				shift = 5;				\
226 				break;					\
227 			}						\
228 		}							\
229 									\
230 		shift;							\
231 	})
232 
ps_to_output_size(unsigned int ps)233 static inline unsigned int ps_to_output_size(unsigned int ps)
234 {
235 	switch (ps) {
236 	case 0: return 32;
237 	case 1: return 36;
238 	case 2: return 40;
239 	case 3: return 42;
240 	case 4: return 44;
241 	case 5:
242 	default:
243 		return 48;
244 	}
245 }
246 
247 #endif /* __ARM64_KVM_NESTED_H */
248