41 #define DA_ALL		(~0)
42 #define DA_ALT		0x01
43 #define DA_RET		0x02
44 #define DA_RETPOLINE	0x04
45 #define DA_ENDBR	0x08
46 #define DA_SMP		0x10
55 	if (!str || kstrtouint(str, 0, &debug_alternative))  in debug_alt()
75 } while (0)
86 		for (j = 0; j < (len) - 1; j++)				\
90 } while (0)
187 	if (insn->opcode.bytes[0] == 0x90 &&  in insn_is_nop()
188 	    (!insn->prefixes.nbytes || insn->prefixes.bytes[0] != 0xF3))  in insn_is_nop()
192 	if (insn->opcode.bytes[0] == 0x0F && insn->opcode.bytes[1] == 0x1F)  in insn_is_nop()
225 	for (int next, i = 0; i < len; i = next) {  in optimize_nops()
293 	} while (0)
320 	for (int next, i = 0; i < instrlen; i = next) {  in __apply_relocation()
328 		switch (insn.opcode.bytes[0]) {  in __apply_relocation()
329 		case 0x0f:  in __apply_relocation()
330 			if (insn.opcode.bytes[1] < 0x80 ||  in __apply_relocation()
331 			    insn.opcode.bytes[1] > 0x8f)  in __apply_relocation()
335 		case 0x70 ... 0x7f:	/* Jcc.d8 */  in __apply_relocation()
348 			if (insn.opcode.bytes[0] == JMP32_INSN_OPCODE) {  in __apply_relocation()
353 					buf[i+0] = JMP8_INSN_OPCODE;  in __apply_relocation()
388 #define CALL_RIP_REL_OPCODE	0xff
389 #define CALL_RIP_REL_MODRM	0x15
402 	if (a->replacementlen != 5 || insn_buff[0] != CALL_INSN_OPCODE) {  in alt_replace_call()
408 	    wr_instr[0] != CALL_RIP_REL_OPCODE ||  in alt_replace_call()
417 	/* ff 15 00 00 00 00   call   *0x0(%rip) */  in alt_replace_call()
421 	/* ff 15 00 00 00 00   call   *0x0 */  in alt_replace_call()
432 		return 0;  in alt_replace_call()
482 		int insn_buff_sz = 0;  in apply_alternatives()
518 		DPRINTK(ALT, "feat: %d*32+%d, old: (%pS (%px) len: %d), repl: (%px, len: %d) flags: 0x%x",  in apply_alternatives()
520 			a->cpuid & 0x1f,  in apply_alternatives()
530 			if (insn_buff_sz < 0)  in apply_alternatives()
535 			insn_buff[insn_buff_sz] = 0x90;  in apply_alternatives()
551 	/* Jcc.d32 second opcode byte is in the range: 0x80-0x8f */  in is_jcc32()
552 	return insn->opcode.bytes[0] == 0x0f && (insn->opcode.bytes[1] & 0xf0) == 0x80;  in is_jcc32()
562 	int i = 0;  in emit_indirect()
567 		modrm = 0x10; /* Reg = 2; CALL r/m */  in emit_indirect()
571 		modrm = 0x20; /* Reg = 4; JMP r/m */  in emit_indirect()
580 		bytes[i++] = 0x41; /* REX.B prefix */  in emit_indirect()
584 	modrm |= 0xc0; /* Mod = 3 */  in emit_indirect()
587 	bytes[i++] = 0xff; /* opcode */  in emit_indirect()
595 	u8 op = insn->opcode.bytes[0];  in emit_call_track_retpoline()
596 	int i = 0;  in emit_call_track_retpoline()
609 		bytes[i++] = 0x2e; /* CS-prefix */  in emit_call_track_retpoline()
656 	int reg, ret, i = 0;  in patch_retpoline()
662 	if (WARN_ON_ONCE(reg & ~0xf))  in patch_retpoline()
676 	op = insn->opcode.bytes[0];  in patch_retpoline()
692 		cc = insn->opcode.bytes[1] & 0xf;  in patch_retpoline()
695 		bytes[i++] = 0x70 + cc;        /* Jcc.d8 */  in patch_retpoline()
706 		bytes[i++] = 0x0f;  in patch_retpoline()
707 		bytes[i++] = 0xae;  in patch_retpoline()
708 		bytes[i++] = 0xe8; /* LFENCE */  in patch_retpoline()
712 	if (ret < 0)  in patch_retpoline()
748 		if (WARN_ON_ONCE(ret < 0))  in apply_retpolines()
751 		op1 = insn.opcode.bytes[0];  in apply_retpolines()
759 		case 0x0f: /* escape */  in apply_retpolines()
760 			if (op2 >= 0x80 && op2 <= 0x8f)  in apply_retpolines()
797 	int i = 0;  in patch_return()
830 		if (WARN_ON_ONCE(ret < 0))  in apply_returns()
833 		op = insn.opcode.bytes[0];  in apply_returns()
949 "	.p2align	2, 0x0					\n"
966 "	.p2align	2, 0x0					\n"
986 		return 0;  in cfi_get_func_hash()
990 		return 0;  in cfi_get_func_hash()
1012 			hash ^= 0x80200003;  in cfi_rehash()
1025 			*next = 0;  in cfi_parse_cmdline()
1047 	return 0;  in cfi_parse_cmdline()
1055  *	movl   $0x12345678,%eax		// 5	     endbr64			// 4
1056  *	nop					     subl   $0x12345678,%r10d   // 7
1070  *	movl	$(-0x12345678),%r10d	 // 6	     movl   $0x12345678,%r10d	// 6
1081 	"	subl	$0x12345678, %r10d	\n"
1097 	"	movl	$0x12345678, %r10d	\n"
1116 	/* b8 78 56 34 12          mov    $0x12345678,%eax */  in decode_preamble_hash()
1117 	if (p[0] == 0xb8)  in decode_preamble_hash()
1120 	return 0; /* invalid hash value */  in decode_preamble_hash()
1127 	/* 41 ba 78 56 34 12       mov    $0x12345678,%r10d */  in decode_caller_hash()
1128 	if (p[0] == 0x41 && p[1] == 0xba)  in decode_caller_hash()
1131 	/* e8 0c 78 56 34 12	   jmp.d8  +12 */  in decode_caller_hash()
1132 	if (p[0] == JMP8_INSN_OPCODE && p[1] == fineibt_caller_jmp)  in decode_caller_hash()
1135 	return 0; /* invalid hash value */  in decode_caller_hash()
1164 	return 0;  in cfi_disable_callers()
1172 	const u8 mov[] = { 0x41, 0xba };  in cfi_enable_callers()
1189 	return 0;  in cfi_enable_callers()
1211 	return 0;  in cfi_rand_preamble()
1229 		WARN_ON(*(u32 *)(wr_addr + fineibt_preamble_hash) != 0x12345678);  in cfi_rewrite_preamble()
1233 	return 0;  in cfi_rewrite_preamble()
1267 	return 0;  in cfi_rand_callers()
1284 			WARN_ON(*(u32 *)(wr_addr + fineibt_caller_hash) != 0x12345678);  in cfi_rewrite_callers()
1290 	return 0;  in cfi_rewrite_callers()
1360 		/* now that nobody targets func()+0, remove ENDBR there */  in __apply_fineibt()
1377 	*(u32 *)addr = 0;  in poison_hash()
1387 		 *	subl	$0, %r10d  in poison_cfi()
1399 		 *	movl	$0, %eax  in poison_cfi()
1400 		 *	.skip	11, 0x90  in poison_cfi()
1442 		if (*ptr == 0x3e)  in alternatives_smp_lock()
1443 			text_poke(ptr, ((unsigned char []){0xf0}), 1);  in alternatives_smp_lock()
1458 		if (*ptr == 0xf0)  in alternatives_smp_unlock()
1459 			text_poke(ptr, ((unsigned char []){0x3E}), 1);  in alternatives_smp_unlock()
1546 		clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP);  in alternatives_enable_smp()
1579 	return 0;  in alternatives_text_reserved()
1643 	unsigned int val = 0;  in int3_selftest()
1681 	 *   lea    0x0(%rip),%rdi  # 5d0: R_X86_64_PC32    .init.data+0x5566c  in alt_reloc_selftest()
1682 	 *   call   +0              # 5d5: R_X86_64_PLT32   __alt_reloc_selftest-0x4  in alt_reloc_selftest()
1908 		pages[0] = vmalloc_to_page(addr);  in __text_poke()
1912 		pages[0] = virt_to_page(addr);  in __text_poke()
1913 		WARN_ON(!PageReserved(pages[0]));  in __text_poke()
1921 	BUG_ON(!pages[0] || (cross_page_boundary && !pages[1]));  in __text_poke()
1941 	pte = mk_pte(pages[0], pgprot);  in __text_poke()
2043 	size_t patched = 0;  in text_poke_copy_locked()
2092 	size_t patched = 0;  in text_poke_set()
2123  * this thing. When len == 6 everything is prefixed with 0x0f and we map
2177 	return 0;  in patch_cmp()
2184 	int ret = 0;  in poke_int3_handler()
2188 		return 0;  in poke_int3_handler()
2196 	 *	write INT3			if (bp_desc.refs != 0)  in poke_int3_handler()
2202 		return 0;  in poke_int3_handler()
2247 	case 0x70 ... 0x7f: /* Jcc */  in poke_int3_handler()
2248 		int3_emulate_jcc(regs, tp->opcode & 0xf, (long)ip, tp->disp);  in poke_int3_handler()
2323 	for (i = 0; i < nr_entries; i++) {  in text_poke_bp_batch()
2333 	for (do_sync = 0, i = 0; i < nr_entries; i++) {  in text_poke_bp_batch()
2339 		if (len - INT3_INSN_SIZE > 0) {  in text_poke_bp_batch()
2345 				_new[0] = 0x0f;  in text_poke_bp_batch()
2397 	for (do_sync = 0, i = 0; i < nr_entries; i++) {  in text_poke_bp_batch()
2398 		u8 byte = tp[i].text[0];  in text_poke_bp_batch()
2401 			byte = 0x0f;  in text_poke_bp_batch()
2424 	int ret, i = 0;  in text_poke_loc_init()
2433 	BUG_ON(ret < 0);  in text_poke_loc_init()
2437 	tp->opcode = insn.opcode.bytes[0];  in text_poke_loc_init()
2443 		tp->opcode = insn.opcode.bytes[1] - 0x10;  in text_poke_loc_init()
2470 	case 0x70 ... 0x7f: /* Jcc */  in text_poke_loc_init()
2476 		case 2: /* NOP2 -- emulate as JMP8+0 */  in text_poke_loc_init()
2479 			tp->disp = 0;  in text_poke_loc_init()
2482 		case 5: /* NOP5 -- emulate as JMP32+0 */  in text_poke_loc_init()
2485 			tp->disp = 0;  in text_poke_loc_init()
2520 		tp_vec_nr = 0;  in text_poke_flush()