1 // SPDX-License-Identifier: GPL-2.0-or-later
19 #include <asm/text-patching.h>
29  * Used to generate warnings if mmu or cpu feature check functions that
82 	start = calc_addr(fcur, fcur->start_off);  in patch_feature_section_mask()
83 	end = calc_addr(fcur, fcur->end_off);  in patch_feature_section_mask()
84 	alt_start = calc_addr(fcur, fcur->alt_start_off);  in patch_feature_section_mask()
85 	alt_end = calc_addr(fcur, fcur->alt_end_off);  in patch_feature_section_mask()
87 	if ((alt_end - alt_start) > (end - start))  in patch_feature_section_mask()
90 	if ((value & fcur->mask & mask) == (fcur->value & mask))  in patch_feature_section_mask()
119 			printk("Unable to patch feature section at %p - %p" \  in do_feature_fixups_mask()
120 				" with %p - %p\n",  in do_feature_fixups_mask()
121 				calc_addr(fcur, fcur->start_off),  in do_feature_fixups_mask()
122 				calc_addr(fcur, fcur->end_off),  in do_feature_fixups_mask()
123 				calc_addr(fcur, fcur->alt_start_off),  in do_feature_fixups_mask()
124 				calc_addr(fcur, fcur->alt_end_off));  in do_feature_fixups_mask()
218 	printk(KERN_DEBUG "stf-barrier: patched %d entry locations (%s barrier)\n", i,  in do_stf_entry_barrier_fixups()
264 	printk(KERN_DEBUG "stf-barrier: patched %d exit locations (%s barrier)\n", i,  in do_stf_exit_barrier_fixups()
289 	 * The call to the fallback entry flush, and the fallback/sync-ori exit  in do_stf_barrier_fixups()
295 	 * The branch to mark interrupt exits non-reentrant is enabled first,  in do_stf_barrier_fixups()
348 	printk(KERN_DEBUG "uaccess-flush: patched %d locations (%s flush)\n", i,  in do_uaccess_flush_fixups()
386 	 * order in which we patch instructions. That's because it's possible we could  in __do_entry_flush_fixups()
390 	 * To make that work, when patching in the fallback flush we patch in this order:  in __do_entry_flush_fixups()
391 	 *  - the mflr		(dest)  in __do_entry_flush_fixups()
392 	 *  - the mtlr		(dest + 2)  in __do_entry_flush_fixups()
393 	 *  - the branch	(dest + 1)  in __do_entry_flush_fixups()
396 	 * patch the mtlr last, it's possible we could return from the branch and not  in __do_entry_flush_fixups()
400 	 * we patch in this order:  in __do_entry_flush_fixups()
401 	 *  - the branch	(dest + 1)  in __do_entry_flush_fixups()
402 	 *  - the mtlr		(dest + 2)  in __do_entry_flush_fixups()
403 	 *  - the mflr		(dest)  in __do_entry_flush_fixups()
406 	 * semi-patched state.  in __do_entry_flush_fixups()
419 	printk(KERN_DEBUG "entry-flush: patched %d locations (%s flush)\n", i,  in __do_entry_flush_fixups()
471 	printk(KERN_DEBUG "rfi-flush: patched %d locations (%s flush)\n", i,  in __do_rfi_flush_fixups()
521 		pr_info("barrier-nospec: using ORI speculation barrier\n");  in do_barrier_nospec_fixups_range()
527 	printk(KERN_DEBUG "barrier-nospec: patched %d locations\n", i);  in do_barrier_nospec_fixups_range()
558 		pr_info("barrier-nospec: using isync; sync as speculation barrier\n");  in do_barrier_nospec_fixups_range()
565 	printk(KERN_DEBUG "barrier-nospec: patched %d locations\n", i);  in do_barrier_nospec_fixups_range()
620 	end = (void *)src + (__end_interrupts - _stext);  in do_final_fixups()
641 	*PTRRELOC(&saved_cpu_features) = spec->cpu_features;  in apply_feature_fixups()
642 	*PTRRELOC(&saved_mmu_features) = spec->mmu_features;  in apply_feature_fixups()
645 	 * Apply the CPU-specific and firmware specific fixups to kernel text  in apply_feature_fixups()
648 	do_feature_fixups(spec->cpu_features,  in apply_feature_fixups()
652 	do_feature_fixups(spec->mmu_features,  in apply_feature_fixups()
656 	do_lwsync_fixups(spec->cpu_features,  in apply_feature_fixups()
671 	saved_mmu_features |= cur_cpu_spec->mmu_features & mask;  in update_mmu_feature_fixups()
673 	do_feature_fixups_mask(cur_cpu_spec->mmu_features, mask,  in update_mmu_feature_fixups()
694 	WARN(saved_cpu_features != cur_cpu_spec->cpu_features,  in check_features()
696 	WARN(saved_mmu_features != cur_cpu_spec->mmu_features,  in check_features()
709 #define check(x)	\  macro
710 	if (!(x)) printk("feature-fixups: test failed at line %d\n", __LINE__);
722 	return (unsigned long)p - (unsigned long)entry;  in calc_offset()
731 	int size = 4 * (end_ftr_fixup_test1 - ftr_fixup_test1);  in test_basic_patching()
738 	/* Sanity check */  in test_basic_patching()
739 	check(memcmp(ftr_fixup_test1, ftr_fixup_test1_orig, size) == 0);  in test_basic_patching()
741 	/* Check we don't patch if the value matches */  in test_basic_patching()
743 	check(memcmp(ftr_fixup_test1, ftr_fixup_test1_orig, size) == 0);  in test_basic_patching()
745 	/* Check we do patch if the value doesn't match */  in test_basic_patching()
747 	check(memcmp(ftr_fixup_test1, ftr_fixup_test1_expected, size) == 0);  in test_basic_patching()
749 	/* Check we do patch if the mask doesn't match */  in test_basic_patching()
751 	check(memcmp(ftr_fixup_test1, ftr_fixup_test1_orig, size) == 0);  in test_basic_patching()
753 	check(memcmp(ftr_fixup_test1, ftr_fixup_test1_expected, size) == 0);  in test_basic_patching()
763 	int size = 4 * (end_ftr_fixup_test2 - ftr_fixup_test2);  in test_alternative_patching()
771 	/* Sanity check */  in test_alternative_patching()
772 	check(memcmp(ftr_fixup_test2, ftr_fixup_test2_orig, size) == 0);  in test_alternative_patching()
774 	/* Check we don't patch if the value matches */  in test_alternative_patching()
776 	check(memcmp(ftr_fixup_test2, ftr_fixup_test2_orig, size) == 0);  in test_alternative_patching()
778 	/* Check we do patch if the value doesn't match */  in test_alternative_patching()
780 	check(memcmp(ftr_fixup_test2, ftr_fixup_test2_expected, size) == 0);  in test_alternative_patching()
782 	/* Check we do patch if the mask doesn't match */  in test_alternative_patching()
784 	check(memcmp(ftr_fixup_test2, ftr_fixup_test2_orig, size) == 0);  in test_alternative_patching()
786 	check(memcmp(ftr_fixup_test2, ftr_fixup_test2_expected, size) == 0);  in test_alternative_patching()
795 	int size = 4 * (end_ftr_fixup_test3 - ftr_fixup_test3);  in test_alternative_case_too_big()
803 	/* Sanity check */  in test_alternative_case_too_big()
804 	check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0);  in test_alternative_case_too_big()
807 	check(patch_feature_section(0xF, &fixup) == 1);  in test_alternative_case_too_big()
808 	check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0);  in test_alternative_case_too_big()
809 	check(patch_feature_section(0, &fixup) == 1);  in test_alternative_case_too_big()
810 	check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0);  in test_alternative_case_too_big()
811 	check(patch_feature_section(~0xF, &fixup) == 1);  in test_alternative_case_too_big()
812 	check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0);  in test_alternative_case_too_big()
822 	int size = 4 * (end_ftr_fixup_test4 - ftr_fixup_test4);  in test_alternative_case_too_small()
825 	/* Check a high-bit flag */  in test_alternative_case_too_small()
826 	flag = 1UL << ((sizeof(unsigned long) - 1) * 8);  in test_alternative_case_too_small()
833 	/* Sanity check */  in test_alternative_case_too_small()
834 	check(memcmp(ftr_fixup_test4, ftr_fixup_test4_orig, size) == 0);  in test_alternative_case_too_small()
836 	/* Check we don't patch if the value matches */  in test_alternative_case_too_small()
838 	check(memcmp(ftr_fixup_test4, ftr_fixup_test4_orig, size) == 0);  in test_alternative_case_too_small()
840 	/* Check we do patch if the value doesn't match */  in test_alternative_case_too_small()
842 	check(memcmp(ftr_fixup_test4, ftr_fixup_test4_expected, size) == 0);  in test_alternative_case_too_small()
844 	/* Check we do patch if the mask doesn't match */  in test_alternative_case_too_small()
846 	check(memcmp(ftr_fixup_test4, ftr_fixup_test4_orig, size) == 0);  in test_alternative_case_too_small()
848 	check(memcmp(ftr_fixup_test4, ftr_fixup_test4_expected, size) == 0);  in test_alternative_case_too_small()
856 	int size = 4 * (end_ftr_fixup_test5 - ftr_fixup_test5);  in test_alternative_case_with_branch()
858 	check(memcmp(ftr_fixup_test5, ftr_fixup_test5_expected, size) == 0);  in test_alternative_case_with_branch()
866 	int size = 4 * (end_ftr_fixup_test6 - ftr_fixup_test6);  in test_alternative_case_with_external_branch()
868 	check(memcmp(ftr_fixup_test6, ftr_fixup_test6_expected, size) == 0);  in test_alternative_case_with_external_branch()
876 	int size = 4 * (end_ftr_fixup_test7 - ftr_fixup_test7);  in test_alternative_case_with_branch_to_end()
878 	check(memcmp(ftr_fixup_test7, ftr_fixup_test7_expected, size) == 0);  in test_alternative_case_with_branch_to_end()
885 	unsigned long size = ftr_fixup_test_FTR_macros_expected -  in test_cpu_macros()
889 	check(memcmp(ftr_fixup_test_FTR_macros,  in test_cpu_macros()
898 	unsigned long size = ftr_fixup_test_FW_FTR_macros_expected -  in test_fw_macros()
902 	check(memcmp(ftr_fixup_test_FW_FTR_macros,  in test_fw_macros()
913 	unsigned long size = end_lwsync_fixup_test -  in test_lwsync_macros()
917 	if (cur_cpu_spec->cpu_features & CPU_FTR_LWSYNC) {  in test_lwsync_macros()
918 		check(memcmp(lwsync_fixup_test,  in test_lwsync_macros()
921 		check(memcmp(lwsync_fixup_test,  in test_lwsync_macros()
933 	int size = sizeof(unsigned int) * (end_ftr_fixup_prefix1 - ftr_fixup_prefix1);  in test_prefix_patching()
940 	/* Sanity check */  in test_prefix_patching()
941 	check(memcmp(ftr_fixup_prefix1, ftr_fixup_prefix1_orig, size) == 0);  in test_prefix_patching()
944 	check(memcmp(ftr_fixup_prefix1, ftr_fixup_prefix1_expected, size) == 0);  in test_prefix_patching()
945 	check(memcmp(ftr_fixup_prefix1, ftr_fixup_prefix1_orig, size) != 0);  in test_prefix_patching()
955 	int size = sizeof(unsigned int) * (end_ftr_fixup_prefix2 - ftr_fixup_prefix2);  in test_prefix_alt_patching()
962 	/* Sanity check */  in test_prefix_alt_patching()
963 	check(memcmp(ftr_fixup_prefix2, ftr_fixup_prefix2_orig, size) == 0);  in test_prefix_alt_patching()
966 	check(memcmp(ftr_fixup_prefix2, ftr_fixup_prefix2_expected, size) == 0);  in test_prefix_alt_patching()
967 	check(memcmp(ftr_fixup_prefix2, ftr_fixup_prefix2_orig, size) != 0);  in test_prefix_alt_patching()
977 	int size = sizeof(unsigned int) * (end_ftr_fixup_prefix3 - ftr_fixup_prefix3);  in test_prefix_word_alt_patching()
984 	/* Sanity check */  in test_prefix_word_alt_patching()
985 	check(memcmp(ftr_fixup_prefix3, ftr_fixup_prefix3_orig, size) == 0);  in test_prefix_word_alt_patching()
988 	check(memcmp(ftr_fixup_prefix3, ftr_fixup_prefix3_expected, size) == 0);  in test_prefix_word_alt_patching()
990 	check(memcmp(ftr_fixup_prefix3, ftr_fixup_prefix3_orig, size) != 0);  in test_prefix_word_alt_patching()
1000 	printk(KERN_DEBUG "Running feature fixup self-tests ...\n");  in test_feature_fixups()