63 static void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool ptw, bool s2)  in fail_s1_walk()  argument
65 	wr->fst		= fst;  in fail_s1_walk()
66 	wr->ptw		= ptw;  in fail_s1_walk()
67 	wr->s2		= s2;  in fail_s1_walk()
68 	wr->failed	= true;  in fail_s1_walk()
149 			 struct s1_walk_result *wr, u64 va)  in setup_s1_walk()  argument
216 			wr->level = S1_MMU_DISABLED;  in setup_s1_walk()
223 			wr->level = S1_MMU_DISABLED;  in setup_s1_walk()
227 	if (wr->level == S1_MMU_DISABLED) {  in setup_s1_walk()
231 		wr->pa = va;  in setup_s1_walk()
348 	fail_s1_walk(wr, ESR_ELx_FSC_ADDRSZ_L(0), false, false);  in setup_s1_walk()
352 	fail_s1_walk(wr, ESR_ELx_FSC_FAULT_L(0), false, false);  in setup_s1_walk()
357 		   struct s1_walk_result *wr, u64 va)  in walk_s1()  argument
381 				fail_s1_walk(wr,  in walk_s1()
388 				fail_s1_walk(wr, ESR_ELx_FSC_PERM_L(level),  in walk_s1()
399 			fail_s1_walk(wr, ESR_ELx_FSC_SEA_TTW(level),  in walk_s1()
423 			wr->APTable  |= FIELD_GET(S1_TABLE_AP, desc);  in walk_s1()
424 			wr->UXNTable |= FIELD_GET(PMD_TABLE_UXN, desc);  in walk_s1()
425 			wr->PXNTable |= FIELD_GET(PMD_TABLE_PXN, desc);  in walk_s1()
462 	wr->failed = false;  in walk_s1()
463 	wr->level = level;  in walk_s1()
464 	wr->desc = desc;  in walk_s1()
465 	wr->pa = desc & GENMASK(47, va_bottom);  in walk_s1()
466 	wr->pa |= va & GENMASK_ULL(va_bottom - 1, 0);  in walk_s1()
471 	fail_s1_walk(wr, ESR_ELx_FSC_ADDRSZ_L(level), true, false);  in walk_s1()
474 	fail_s1_walk(wr, ESR_ELx_FSC_FAULT_L(level), true, false);  in walk_s1()
781 static u64 compute_par_s1(struct kvm_vcpu *vcpu, struct s1_walk_result *wr,  in compute_par_s1()  argument
786 	if (wr->failed) {  in compute_par_s1()
789 		par |= FIELD_PREP(SYS_PAR_EL1_FST, wr->fst);  in compute_par_s1()
790 		par |= wr->ptw ? SYS_PAR_EL1_PTW : 0;  in compute_par_s1()
791 		par |= wr->s2 ? SYS_PAR_EL1_S : 0;  in compute_par_s1()
792 	} else if (wr->level == S1_MMU_DISABLED) {  in compute_par_s1()
795 		par |= wr->pa & GENMASK_ULL(47, 12);  in compute_par_s1()
816 		mair >>= FIELD_GET(PTE_ATTRINDX_MASK, wr->desc) * 8;  in compute_par_s1()
828 		par |= wr->pa & GENMASK_ULL(47, 12);  in compute_par_s1()
830 		sh = compute_sh(mair, wr->desc);  in compute_par_s1()
857 					  struct s1_walk_result *wr)  in compute_s1_direct_permissions()  argument
863 		switch (FIELD_GET(PTE_USER | PTE_RDONLY, wr->desc)) {  in compute_s1_direct_permissions()
865 			wr->pr = wr->pw = true;  in compute_s1_direct_permissions()
866 			wr->ur = wr->uw = false;  in compute_s1_direct_permissions()
869 			wr->pr = wr->pw = wr->ur = wr->uw = true;  in compute_s1_direct_permissions()
872 			wr->pr = true;  in compute_s1_direct_permissions()
873 			wr->pw = wr->ur = wr->uw = false;  in compute_s1_direct_permissions()
876 			wr->pr = wr->ur = true;  in compute_s1_direct_permissions()
877 			wr->pw = wr->uw = false;  in compute_s1_direct_permissions()
882 		wr->px = !((wr->desc & PTE_PXN) || wr->uw);  in compute_s1_direct_permissions()
883 		wr->ux = !(wr->desc & PTE_UXN);  in compute_s1_direct_permissions()
885 		wr->ur = wr->uw = wr->ux = false;  in compute_s1_direct_permissions()
887 		if (!(wr->desc & PTE_RDONLY)) {  in compute_s1_direct_permissions()
888 			wr->pr = wr->pw = true;  in compute_s1_direct_permissions()
890 			wr->pr = true;  in compute_s1_direct_permissions()
891 			wr->pw = false;  in compute_s1_direct_permissions()
895 		wr->px = !(wr->desc & PTE_UXN);  in compute_s1_direct_permissions()
908 	wr->pwxn = wr->uwxn = wxn;  in compute_s1_direct_permissions()
909 	wr->pov = wi->poe;  in compute_s1_direct_permissions()
910 	wr->uov = wi->e0poe;  in compute_s1_direct_permissions()
915 						struct s1_walk_result *wr)  in compute_s1_hierarchical_permissions()  argument
919 		switch (wr->APTable) {  in compute_s1_hierarchical_permissions()
923 			wr->ur = wr->uw = false;  in compute_s1_hierarchical_permissions()
926 			wr->pw = wr->uw = false;  in compute_s1_hierarchical_permissions()
929 			wr->pw = wr->ur = wr->uw = false;  in compute_s1_hierarchical_permissions()
933 		wr->px &= !wr->PXNTable;  in compute_s1_hierarchical_permissions()
934 		wr->ux &= !wr->UXNTable;  in compute_s1_hierarchical_permissions()
936 		if (wr->APTable & BIT(1))  in compute_s1_hierarchical_permissions()
937 			wr->pw = false;  in compute_s1_hierarchical_permissions()
940 		wr->px &= !wr->UXNTable;  in compute_s1_hierarchical_permissions()
946 #define set_priv_perms(wr, r, w, x)	\  argument
948 		(wr)->pr = (r);		\
949 		(wr)->pw = (w);		\
950 		(wr)->px = (x);		\
953 #define set_unpriv_perms(wr, r, w, x)	\  argument
955 		(wr)->ur = (r);		\
956 		(wr)->uw = (w);		\
957 		(wr)->ux = (x);		\
960 #define set_priv_wxn(wr, v)		\  argument
962 		(wr)->pwxn = (v);	\
965 #define set_unpriv_wxn(wr, v)		\  argument
967 		(wr)->uwxn = (v);	\
971 #define set_perms(w, wr, ip)						\  argument
976 			set_ ## w ## _perms((wr), false, false, false);	\
979 			set_ ## w ## _perms((wr), true , false, false);	\
982 			set_ ## w ## _perms((wr), false, false, true );	\
985 			set_ ## w ## _perms((wr), true , false, true );	\
988 			set_ ## w ## _perms((wr), false, false, false);	\
991 			set_ ## w ## _perms((wr), true , true , false);	\
994 			set_ ## w ## _perms((wr), true , true , true );	\
997 			set_ ## w ## _perms((wr), true , true , true );	\
1000 			set_ ## w ## _perms((wr), true , false, false);	\
1003 			set_ ## w ## _perms((wr), true , false, false);	\
1006 			set_ ## w ## _perms((wr), true , false, true );	\
1009 			set_ ## w ## _perms((wr), false, false, false);	\
1012 			set_ ## w ## _perms((wr), true , true , false);	\
1015 			set_ ## w ## _perms((wr), false, false, false);	\
1018 			set_ ## w ## _perms((wr), true , true , true );	\
1021 			set_ ## w ## _perms((wr), false, false, false);	\
1026 		set_ ## w ## _wxn((wr), ((ip) == 0b0110));		\
1032 					    struct s1_walk_result *wr)  in compute_s1_indirect_permissions()  argument
1036 	idx = pte_pi_index(wr->desc);  in compute_s1_indirect_permissions()
1053 	set_perms(priv, wr, pp);  in compute_s1_indirect_permissions()
1056 		set_perms(unpriv, wr, up);  in compute_s1_indirect_permissions()
1058 		set_unpriv_perms(wr, false, false, false);  in compute_s1_indirect_permissions()
1060 	wr->pov = wi->poe && !(pp & BIT(3));  in compute_s1_indirect_permissions()
1061 	wr->uov = wi->e0poe && !(up & BIT(3));  in compute_s1_indirect_permissions()
1064 	if (wr->px && wr->uw) {  in compute_s1_indirect_permissions()
1065 		set_priv_perms(wr, false, false, false);  in compute_s1_indirect_permissions()
1066 		set_unpriv_perms(wr, false, false, false);  in compute_s1_indirect_permissions()
1072 					   struct s1_walk_result *wr)  in compute_s1_overlay_permissions()  argument
1076 	idx = FIELD_GET(PTE_PO_IDX_MASK, wr->desc);  in compute_s1_overlay_permissions()
1096 	if (wi->poe && wr->pov) {  in compute_s1_overlay_permissions()
1097 		wr->pr &= pov_perms & POE_R;  in compute_s1_overlay_permissions()
1098 		wr->px &= pov_perms & POE_X;  in compute_s1_overlay_permissions()
1099 		wr->pw &= pov_perms & POE_W;  in compute_s1_overlay_permissions()
1105 	if (wi->e0poe && wr->uov) {  in compute_s1_overlay_permissions()
1106 		wr->ur &= uov_perms & POE_R;  in compute_s1_overlay_permissions()
1107 		wr->ux &= uov_perms & POE_X;  in compute_s1_overlay_permissions()
1108 		wr->uw &= uov_perms & POE_W;  in compute_s1_overlay_permissions()
1114 				   struct s1_walk_result *wr)  in compute_s1_permissions()  argument
1119 		compute_s1_direct_permissions(vcpu, wi, wr);  in compute_s1_permissions()
1121 		compute_s1_indirect_permissions(vcpu, wi, wr);  in compute_s1_permissions()
1124 		compute_s1_hierarchical_permissions(vcpu, wi, wr);  in compute_s1_permissions()
1127 		compute_s1_overlay_permissions(vcpu, wi, wr);  in compute_s1_permissions()
1130 	if (wr->pwxn) {  in compute_s1_permissions()
1131 		if (!wr->pov && wr->pw)  in compute_s1_permissions()
1132 			wr->px = false;  in compute_s1_permissions()
1133 		if (wr->pov && wr->px)  in compute_s1_permissions()
1134 			wr->pw = false;  in compute_s1_permissions()
1138 	if (wr->uwxn) {  in compute_s1_permissions()
1139 		if (!wr->uov && wr->uw)  in compute_s1_permissions()
1140 			wr->ux = false;  in compute_s1_permissions()
1141 		if (wr->uov && wr->ux)  in compute_s1_permissions()
1142 			wr->uw = false;  in compute_s1_permissions()
1145 	pan = wi->pan && (wr->ur || wr->uw ||  in compute_s1_permissions()
1146 			  (pan3_enabled(vcpu, wi->regime) && wr->ux));  in compute_s1_permissions()
1147 	wr->pw &= !pan;  in compute_s1_permissions()
1148 	wr->pr &= !pan;  in compute_s1_permissions()
1153 	struct s1_walk_result wr = {};  in handle_at_slow()  local
1158 	ret = setup_s1_walk(vcpu, op, &wi, &wr, vaddr);  in handle_at_slow()
1162 	if (wr.level == S1_MMU_DISABLED)  in handle_at_slow()
1167 	ret = walk_s1(vcpu, &wi, &wr, vaddr);  in handle_at_slow()
1174 	compute_s1_permissions(vcpu, &wi, &wr);  in handle_at_slow()
1180 		perm_fail = !wr.pr;  in handle_at_slow()
1185 		perm_fail = !wr.pw;  in handle_at_slow()
1188 		perm_fail = !wr.ur;  in handle_at_slow()
1191 		perm_fail = !wr.uw;  in handle_at_slow()
1201 		fail_s1_walk(&wr, ESR_ELx_FSC_PERM_L(wr.level), false, false);  in handle_at_slow()
1204 	return compute_par_s1(vcpu, &wr, wi.regime);  in handle_at_slow()