Lines Matching +full:op +full:- +full:mode
1 // SPDX-License-Identifier: GPL-2.0-only
5 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
9 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
18 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
27 #include <asm/nospec-branch.h>
44 #define OpMem64 6ull /* Memory, 64-bit */
45 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
48 #define OpImmByte 10ull /* 8-bit sign extended immediate */
50 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
51 #define OpMem16 13ull /* Memory operand (16-bit). */
52 #define OpMem32 14ull /* Memory operand (32-bit). */
64 #define OpMem8 26ull /* 8-bit zero extended memory operand */
65 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
66 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
68 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
71 #define OpMask ((1ull << OpBits) - 1)
74 * Opcode effective-address decode tables.
82 /* Operand sizes: 8-bit operands or specified/overridden size. */
83 #define ByteOp (1<<0) /* 8-bit operands. */
138 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
141 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
223 struct opcode op[8]; member
248 unsigned long dirty = ctxt->regs_dirty; in writeback_registers()
252 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]); in writeback_registers()
257 ctxt->regs_dirty = 0; in invalidate_registers()
258 ctxt->regs_valid = 0; in invalidate_registers()
307 ".size " name ", .-" name "\n\t"
312 #define __FOP_START(op, align) \ argument
313 extern void em_##op(struct fastop *fake); \
315 ".global em_" #op " \n\t" \
317 "em_" #op ":\n\t"
319 #define FOP_START(op) __FOP_START(op, FASTOP_SIZE) argument
331 #define FOP1E(op, dst) \ argument
332 __FOP_FUNC(#op "_" #dst) \
333 "10: " #op " %" #dst " \n\t" \
334 __FOP_RET(#op "_" #dst)
336 #define FOP1EEX(op, dst) \ argument
337 FOP1E(op, dst) _ASM_EXTABLE_TYPE_REG(10b, 11b, EX_TYPE_ZERO_REG, %%esi)
339 #define FASTOP1(op) \ argument
340 FOP_START(op) \
341 FOP1E(op##b, al) \
342 FOP1E(op##w, ax) \
343 FOP1E(op##l, eax) \
344 ON64(FOP1E(op##q, rax)) \
347 /* 1-operand, using src2 (for MUL/DIV r/m) */
348 #define FASTOP1SRC2(op, name) \ argument
350 FOP1E(op, cl) \
351 FOP1E(op, cx) \
352 FOP1E(op, ecx) \
353 ON64(FOP1E(op, rcx)) \
356 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
357 #define FASTOP1SRC2EX(op, name) \ argument
359 FOP1EEX(op, cl) \
360 FOP1EEX(op, cx) \
361 FOP1EEX(op, ecx) \
362 ON64(FOP1EEX(op, rcx)) \
365 #define FOP2E(op, dst, src) \ argument
366 __FOP_FUNC(#op "_" #dst "_" #src) \
367 #op " %" #src ", %" #dst " \n\t" \
368 __FOP_RET(#op "_" #dst "_" #src)
370 #define FASTOP2(op) \ argument
371 FOP_START(op) \
372 FOP2E(op##b, al, dl) \
373 FOP2E(op##w, ax, dx) \
374 FOP2E(op##l, eax, edx) \
375 ON64(FOP2E(op##q, rax, rdx)) \
379 #define FASTOP2W(op) \ argument
380 FOP_START(op) \
382 FOP2E(op##w, ax, dx) \
383 FOP2E(op##l, eax, edx) \
384 ON64(FOP2E(op##q, rax, rdx)) \
388 #define FASTOP2CL(op) \ argument
389 FOP_START(op) \
390 FOP2E(op##b, al, cl) \
391 FOP2E(op##w, ax, cl) \
392 FOP2E(op##l, eax, cl) \
393 ON64(FOP2E(op##q, rax, cl)) \
397 #define FASTOP2R(op, name) \ argument
399 FOP2E(op##b, dl, al) \
400 FOP2E(op##w, dx, ax) \
401 FOP2E(op##l, edx, eax) \
402 ON64(FOP2E(op##q, rdx, rax)) \
405 #define FOP3E(op, dst, src, src2) \ argument
406 __FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
407 #op " %" #src2 ", %" #src ", %" #dst " \n\t"\
408 __FOP_RET(#op "_" #dst "_" #src "_" #src2)
410 /* 3-operand, word-only, src2=cl */
411 #define FASTOP3WCL(op) \ argument
412 FOP_START(op) \
414 FOP3E(op##w, ax, dx, cl) \
415 FOP3E(op##l, eax, edx, cl) \
416 ON64(FOP3E(op##q, rax, rdx, cl)) \
419 /* Special case for SETcc - 1 instruction per cc */
420 #define FOP_SETCC(op) \ argument
421 FOP_FUNC(op) \
422 #op " %al \n\t" \
423 FOP_RET(op)
472 .rep_prefix = ctxt->rep_prefix, in emulator_check_intercept()
473 .modrm_mod = ctxt->modrm_mod, in emulator_check_intercept()
474 .modrm_reg = ctxt->modrm_reg, in emulator_check_intercept()
475 .modrm_rm = ctxt->modrm_rm, in emulator_check_intercept()
476 .src_val = ctxt->src.val64, in emulator_check_intercept()
477 .dst_val = ctxt->dst.val64, in emulator_check_intercept()
478 .src_bytes = ctxt->src.bytes, in emulator_check_intercept()
479 .dst_bytes = ctxt->dst.bytes, in emulator_check_intercept()
480 .ad_bytes = ctxt->ad_bytes, in emulator_check_intercept()
481 .next_rip = ctxt->eip, in emulator_check_intercept()
484 return ctxt->ops->intercept(ctxt, &info, stage); in emulator_check_intercept()
494 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */ in assign_register()
504 break; /* 64b: zero-extend */ in assign_register()
513 return (1UL << (ctxt->ad_bytes << 3)) - 1; in ad_mask()
521 if (ctxt->mode == X86EMUL_MODE_PROT64) in stack_mask()
523 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS); in stack_mask()
532 /* Access/update address held in a register, based on addressing mode. */
536 if (ctxt->ad_bytes == sizeof(unsigned long)) in address_mask()
558 assign_register(preg, *preg + inc, ctxt->ad_bytes); in register_address_increment()
570 return desc->g ? (limit << 12) | 0xfff : limit; in desc_limit_scaled()
575 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS) in seg_base()
578 return ctxt->ops->get_cached_segment_base(ctxt, seg); in seg_base()
587 ctxt->exception.vector = vec; in emulate_exception()
588 ctxt->exception.error_code = error; in emulate_exception()
589 ctxt->exception.error_code_valid = valid; in emulate_exception()
633 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg); in get_segment_selector()
644 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg); in set_segment_selector()
645 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg); in set_segment_selector()
650 return (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_LA57) ? 57 : 48; in ctxt_virt_addr_bits()
657 return !ctxt->ops->is_canonical_addr(ctxt, la, flags); in emul_is_noncanonical_address()
671 u64 alignment = ctxt->d & AlignMask; in insn_alignment()
691 enum x86emul_mode mode, ulong *linear, in __linearize() argument
703 switch (mode) { in __linearize()
705 *linear = la = ctxt->ops->get_untagged_addr(ctxt, la, flags); in __linearize()
710 *max_size = min_t(u64, ~0u, (1ull << va_bits) - la); in __linearize()
716 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL, in __linearize()
720 /* code segment in protected mode or read-only data segment */ in __linearize()
721 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8)) || !(desc.type & 2)) && in __linearize()
729 /* expand-down segment */ in __linearize()
739 *max_size = (u64)lim + 1 - addr.ea; in __linearize()
745 if (la & (insn_alignment(ctxt, size) - 1)) in __linearize()
761 return __linearize(ctxt, addr, &max_size, size, ctxt->mode, linear, in linearize()
773 if (ctxt->op_bytes != sizeof(unsigned long)) in assign_eip()
774 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1); in assign_eip()
775 rc = __linearize(ctxt, addr, &max_size, 1, ctxt->mode, &linear, in assign_eip()
778 ctxt->_eip = addr.ea; in assign_eip()
789 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer); in emulator_recalc_and_set_mode()
791 if (!(ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PE)) { in emulator_recalc_and_set_mode()
792 /* Real mode. cpu must not have long mode active */ in emulator_recalc_and_set_mode()
795 ctxt->mode = X86EMUL_MODE_REAL; in emulator_recalc_and_set_mode()
799 if (ctxt->eflags & X86_EFLAGS_VM) { in emulator_recalc_and_set_mode()
800 /* Protected/VM86 mode. cpu must not have long mode active */ in emulator_recalc_and_set_mode()
803 ctxt->mode = X86EMUL_MODE_VM86; in emulator_recalc_and_set_mode()
807 if (!ctxt->ops->get_segment(ctxt, &selector, &cs, &base3, VCPU_SREG_CS)) in emulator_recalc_and_set_mode()
812 /* Proper long mode */ in emulator_recalc_and_set_mode()
813 ctxt->mode = X86EMUL_MODE_PROT64; in emulator_recalc_and_set_mode()
815 /* 32 bit compatibility mode*/ in emulator_recalc_and_set_mode()
816 ctxt->mode = X86EMUL_MODE_PROT32; in emulator_recalc_and_set_mode()
818 ctxt->mode = X86EMUL_MODE_PROT16; in emulator_recalc_and_set_mode()
821 /* Legacy 32 bit / 16 bit mode */ in emulator_recalc_and_set_mode()
822 ctxt->mode = cs.d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16; in emulator_recalc_and_set_mode()
845 return assign_eip_near(ctxt, ctxt->_eip + rel); in jmp_rel()
851 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true); in linear_read_system()
858 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true); in linear_write_system()
872 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false); in segmented_read_std()
886 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false); in segmented_write_std()
898 int cur_size = ctxt->fetch.end - ctxt->fetch.data; in __do_insn_fetch_bytes()
900 .ea = ctxt->eip + cur_size }; in __do_insn_fetch_bytes()
912 rc = __linearize(ctxt, addr, &max_size, 0, ctxt->mode, &linear, in __do_insn_fetch_bytes()
918 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear)); in __do_insn_fetch_bytes()
924 * still, we must have hit the 15-byte boundary. in __do_insn_fetch_bytes()
929 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end, in __do_insn_fetch_bytes()
930 size, &ctxt->exception); in __do_insn_fetch_bytes()
933 ctxt->fetch.end += size; in __do_insn_fetch_bytes()
940 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr; in do_insn_fetch_bytes()
943 return __do_insn_fetch_bytes(ctxt, size - done_size); in do_insn_fetch_bytes()
955 ctxt->_eip += sizeof(_type); \
956 memcpy(&_x, ctxt->fetch.ptr, sizeof(_type)); \
957 ctxt->fetch.ptr += sizeof(_type); \
966 ctxt->_eip += (_size); \
967 memcpy(_arr, ctxt->fetch.ptr, _size); \
968 ctxt->fetch.ptr += (_size); \
980 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop; in decode_register()
1053 if (ctxt->src.val == 0) in em_bsf_c()
1054 ctxt->dst.type = OP_NONE; in em_bsf_c()
1061 if (ctxt->src.val == 0) in em_bsr_c()
1062 ctxt->dst.type = OP_NONE; in em_bsr_c()
1077 static void fetch_register_operand(struct operand *op) in fetch_register_operand() argument
1079 switch (op->bytes) { in fetch_register_operand()
1081 op->val = *(u8 *)op->addr.reg; in fetch_register_operand()
1084 op->val = *(u16 *)op->addr.reg; in fetch_register_operand()
1087 op->val = *(u32 *)op->addr.reg; in fetch_register_operand()
1090 op->val = *(u64 *)op->addr.reg; in fetch_register_operand()
1097 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM)) in em_fninit()
1110 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM)) in em_fnstcw()
1117 ctxt->dst.val = fcw; in em_fnstcw()
1126 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM)) in em_fnstsw()
1133 ctxt->dst.val = fsw; in em_fnstsw()
1139 struct operand *op) in decode_register_operand() argument
1143 if (ctxt->d & ModRM) in decode_register_operand()
1144 reg = ctxt->modrm_reg; in decode_register_operand()
1146 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3); in decode_register_operand()
1148 if (ctxt->d & Sse) { in decode_register_operand()
1149 op->type = OP_XMM; in decode_register_operand()
1150 op->bytes = 16; in decode_register_operand()
1151 op->addr.xmm = reg; in decode_register_operand()
1152 kvm_read_sse_reg(reg, &op->vec_val); in decode_register_operand()
1155 if (ctxt->d & Mmx) { in decode_register_operand()
1157 op->type = OP_MM; in decode_register_operand()
1158 op->bytes = 8; in decode_register_operand()
1159 op->addr.mm = reg; in decode_register_operand()
1163 op->type = OP_REG; in decode_register_operand()
1164 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; in decode_register_operand()
1165 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp); in decode_register_operand()
1167 fetch_register_operand(op); in decode_register_operand()
1168 op->orig_val = op->val; in decode_register_operand()
1174 ctxt->modrm_seg = VCPU_SREG_SS; in adjust_modrm_seg()
1178 struct operand *op) in decode_modrm() argument
1185 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */ in decode_modrm()
1186 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */ in decode_modrm()
1187 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */ in decode_modrm()
1189 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6; in decode_modrm()
1190 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3; in decode_modrm()
1191 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07); in decode_modrm()
1192 ctxt->modrm_seg = VCPU_SREG_DS; in decode_modrm()
1194 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) { in decode_modrm()
1195 op->type = OP_REG; in decode_modrm()
1196 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; in decode_modrm()
1197 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm, in decode_modrm()
1198 ctxt->d & ByteOp); in decode_modrm()
1199 if (ctxt->d & Sse) { in decode_modrm()
1200 op->type = OP_XMM; in decode_modrm()
1201 op->bytes = 16; in decode_modrm()
1202 op->addr.xmm = ctxt->modrm_rm; in decode_modrm()
1203 kvm_read_sse_reg(ctxt->modrm_rm, &op->vec_val); in decode_modrm()
1206 if (ctxt->d & Mmx) { in decode_modrm()
1207 op->type = OP_MM; in decode_modrm()
1208 op->bytes = 8; in decode_modrm()
1209 op->addr.mm = ctxt->modrm_rm & 7; in decode_modrm()
1212 fetch_register_operand(op); in decode_modrm()
1216 op->type = OP_MEM; in decode_modrm()
1218 if (ctxt->ad_bytes == 2) { in decode_modrm()
1224 /* 16-bit ModR/M decode. */ in decode_modrm()
1225 switch (ctxt->modrm_mod) { in decode_modrm()
1227 if (ctxt->modrm_rm == 6) in decode_modrm()
1237 switch (ctxt->modrm_rm) { in decode_modrm()
1257 if (ctxt->modrm_mod != 0) in decode_modrm()
1264 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 || in decode_modrm()
1265 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0)) in decode_modrm()
1266 ctxt->modrm_seg = VCPU_SREG_SS; in decode_modrm()
1269 /* 32/64-bit ModR/M decode. */ in decode_modrm()
1270 if ((ctxt->modrm_rm & 7) == 4) { in decode_modrm()
1276 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0) in decode_modrm()
1282 if ((ctxt->d & IncSP) && in decode_modrm()
1284 modrm_ea += ctxt->op_bytes; in decode_modrm()
1288 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) { in decode_modrm()
1290 if (ctxt->mode == X86EMUL_MODE_PROT64) in decode_modrm()
1291 ctxt->rip_relative = 1; in decode_modrm()
1293 base_reg = ctxt->modrm_rm; in decode_modrm()
1297 switch (ctxt->modrm_mod) { in decode_modrm()
1306 op->addr.mem.ea = modrm_ea; in decode_modrm()
1307 if (ctxt->ad_bytes != 8) in decode_modrm()
1308 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea; in decode_modrm()
1315 struct operand *op) in decode_abs() argument
1319 op->type = OP_MEM; in decode_abs()
1320 switch (ctxt->ad_bytes) { in decode_abs()
1322 op->addr.mem.ea = insn_fetch(u16, ctxt); in decode_abs()
1325 op->addr.mem.ea = insn_fetch(u32, ctxt); in decode_abs()
1328 op->addr.mem.ea = insn_fetch(u64, ctxt); in decode_abs()
1339 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) { in fetch_bit_operand()
1340 mask = ~((long)ctxt->dst.bytes * 8 - 1); in fetch_bit_operand()
1342 if (ctxt->src.bytes == 2) in fetch_bit_operand()
1343 sv = (s16)ctxt->src.val & (s16)mask; in fetch_bit_operand()
1344 else if (ctxt->src.bytes == 4) in fetch_bit_operand()
1345 sv = (s32)ctxt->src.val & (s32)mask; in fetch_bit_operand()
1347 sv = (s64)ctxt->src.val & (s64)mask; in fetch_bit_operand()
1349 ctxt->dst.addr.mem.ea = address_mask(ctxt, in fetch_bit_operand()
1350 ctxt->dst.addr.mem.ea + (sv >> 3)); in fetch_bit_operand()
1354 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1; in fetch_bit_operand()
1361 struct read_cache *mc = &ctxt->mem_read; in read_emulated()
1363 if (mc->pos < mc->end) in read_emulated()
1366 if (KVM_EMULATOR_BUG_ON((mc->end + size) >= sizeof(mc->data), ctxt)) in read_emulated()
1369 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size, in read_emulated()
1370 &ctxt->exception); in read_emulated()
1374 mc->end += size; in read_emulated()
1377 memcpy(dest, mc->data + mc->pos, size); in read_emulated()
1378 mc->pos += size; in read_emulated()
1407 return ctxt->ops->write_emulated(ctxt, linear, data, size, in segmented_write()
1408 &ctxt->exception); in segmented_write()
1422 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data, in segmented_cmpxchg()
1423 size, &ctxt->exception); in segmented_cmpxchg()
1430 struct read_cache *rc = &ctxt->io_read; in pio_in_emulated()
1432 if (rc->pos == rc->end) { /* refill pio read ahead */ in pio_in_emulated()
1434 unsigned int count = ctxt->rep_prefix ? in pio_in_emulated()
1436 in_page = (ctxt->eflags & X86_EFLAGS_DF) ? in pio_in_emulated()
1438 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)); in pio_in_emulated()
1439 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count); in pio_in_emulated()
1442 rc->pos = rc->end = 0; in pio_in_emulated()
1443 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n)) in pio_in_emulated()
1445 rc->end = n * size; in pio_in_emulated()
1448 if (ctxt->rep_prefix && (ctxt->d & String) && in pio_in_emulated()
1449 !(ctxt->eflags & X86_EFLAGS_DF)) { in pio_in_emulated()
1450 ctxt->dst.data = rc->data + rc->pos; in pio_in_emulated()
1451 ctxt->dst.type = OP_MEM_STR; in pio_in_emulated()
1452 ctxt->dst.count = (rc->end - rc->pos) / size; in pio_in_emulated()
1453 rc->pos = rc->end; in pio_in_emulated()
1455 memcpy(dest, rc->data + rc->pos, size); in pio_in_emulated()
1456 rc->pos += size; in pio_in_emulated()
1467 ctxt->ops->get_idt(ctxt, &dt); in read_interrupt_descriptor()
1479 const struct x86_emulate_ops *ops = ctxt->ops; in get_descriptor_table_ptr()
1487 if (!ops->get_segment(ctxt, &sel, &desc, &base3, in get_descriptor_table_ptr()
1491 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */ in get_descriptor_table_ptr()
1492 dt->address = get_desc_base(&desc) | ((u64)base3 << 32); in get_descriptor_table_ptr()
1494 ops->get_gdt(ctxt, dt); in get_descriptor_table_ptr()
1515 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer); in get_descriptor_ptr()
1517 addr &= (u32)-1; in get_descriptor_ptr()
1562 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */ in __load_segment_descriptor()
1570 if (ctxt->mode == X86EMUL_MODE_REAL) { in __load_segment_descriptor()
1571 /* set real mode segment descriptor (keep limit etc. for in __load_segment_descriptor()
1572 * unreal mode) */ in __load_segment_descriptor()
1573 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg); in __load_segment_descriptor()
1576 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) { in __load_segment_descriptor()
1593 /* NULL selector is not valid for TR, CS and (except for long mode) SS */ in __load_segment_descriptor()
1599 if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl) in __load_segment_descriptor()
1603 * ctxt->ops->set_segment expects the CPL to be in in __load_segment_descriptor()
1604 * SS.DPL, so fake an expand-up 32-bit data segment. in __load_segment_descriptor()
1647 * Mode exceptions and IRET (handled above). In all other in __load_segment_descriptor()
1660 /* Outer-privilege level return is not implemented */ in __load_segment_descriptor()
1685 /* in long-mode d/b must be clear if l is set */ in __load_segment_descriptor()
1689 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer); in __load_segment_descriptor()
1694 /* CS(RPL) <- CPL */ in __load_segment_descriptor()
1732 } else if (ctxt->mode == X86EMUL_MODE_PROT64) { in __load_segment_descriptor()
1745 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc, in __load_segment_descriptor()
1746 sizeof(seg_desc), &ctxt->exception); in __load_segment_descriptor()
1751 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg); in __load_segment_descriptor()
1762 u8 cpl = ctxt->ops->cpl(ctxt); in load_segment_descriptor()
1775 ctxt->mode == X86EMUL_MODE_PROT64) in load_segment_descriptor()
1782 static void write_register_operand(struct operand *op) in write_register_operand() argument
1784 return assign_register(op->addr.reg, op->val, op->bytes); in write_register_operand()
1787 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op) in writeback() argument
1789 switch (op->type) { in writeback()
1791 write_register_operand(op); in writeback()
1794 if (ctxt->lock_prefix) in writeback()
1796 op->addr.mem, in writeback()
1797 &op->orig_val, in writeback()
1798 &op->val, in writeback()
1799 op->bytes); in writeback()
1802 op->addr.mem, in writeback()
1803 &op->val, in writeback()
1804 op->bytes); in writeback()
1807 op->addr.mem, in writeback()
1808 op->data, in writeback()
1809 op->bytes * op->count); in writeback()
1811 kvm_write_sse_reg(op->addr.xmm, &op->vec_val); in writeback()
1814 kvm_write_mmx_reg(op->addr.mm, &op->mm_val); in writeback()
1829 rsp_increment(ctxt, -len); in emulate_push()
1839 ctxt->dst.type = OP_NONE; in em_push()
1840 return emulate_push(ctxt, &ctxt->src.val, ctxt->op_bytes); in em_push()
1861 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes); in em_pop()
1870 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT; in emulate_popf()
1871 int cpl = ctxt->ops->cpl(ctxt); in emulate_popf()
1882 switch(ctxt->mode) { in emulate_popf()
1896 default: /* real mode */ in emulate_popf()
1902 (ctxt->eflags & ~change_mask) | (val & change_mask); in emulate_popf()
1909 ctxt->dst.type = OP_REG; in em_popf()
1910 ctxt->dst.addr.reg = &ctxt->eflags; in em_popf()
1911 ctxt->dst.bytes = ctxt->op_bytes; in em_popf()
1912 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes); in em_popf()
1918 unsigned frame_size = ctxt->src.val; in em_enter()
1919 unsigned nesting_level = ctxt->src2.val & 31; in em_enter()
1932 reg_read(ctxt, VCPU_REGS_RSP) - frame_size, in em_enter()
1941 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes); in em_leave()
1946 int seg = ctxt->src2.val; in em_push_sreg()
1948 ctxt->src.val = get_segment_selector(ctxt, seg); in em_push_sreg()
1949 if (ctxt->op_bytes == 4) { in em_push_sreg()
1950 rsp_increment(ctxt, -2); in em_push_sreg()
1951 ctxt->op_bytes = 2; in em_push_sreg()
1959 int seg = ctxt->src2.val; in em_pop_sreg()
1968 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS; in em_pop_sreg()
1969 if (ctxt->op_bytes > 2) in em_pop_sreg()
1970 rsp_increment(ctxt, ctxt->op_bytes - 2); in em_pop_sreg()
1984 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg)); in em_pusha()
1998 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM; in em_pushf()
2010 rsp_increment(ctxt, ctxt->op_bytes); in em_popa()
2011 --reg; in em_popa()
2014 rc = emulate_pop(ctxt, &val, ctxt->op_bytes); in em_popa()
2017 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes); in em_popa()
2018 --reg; in em_popa()
2025 const struct x86_emulate_ops *ops = ctxt->ops; in __emulate_int_real()
2033 ctxt->src.val = ctxt->eflags; in __emulate_int_real()
2038 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC); in __emulate_int_real()
2040 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS); in __emulate_int_real()
2045 ctxt->src.val = ctxt->_eip; in __emulate_int_real()
2050 ops->get_idt(ctxt, &dt); in __emulate_int_real()
2067 ctxt->_eip = eip; in __emulate_int_real()
2085 switch(ctxt->mode) { in emulate_int()
2093 /* Protected mode interrupts unimplemented yet */ in emulate_int()
2115 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes); in emulate_iret_real()
2123 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes); in emulate_iret_real()
2128 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes); in emulate_iret_real()
2138 ctxt->_eip = temp_eip; in emulate_iret_real()
2140 if (ctxt->op_bytes == 4) in emulate_iret_real()
2141 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask)); in emulate_iret_real()
2142 else if (ctxt->op_bytes == 2) { in emulate_iret_real()
2143 ctxt->eflags &= ~0xffff; in emulate_iret_real()
2144 ctxt->eflags |= temp_eflags; in emulate_iret_real()
2147 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */ in emulate_iret_real()
2148 ctxt->eflags |= X86_EFLAGS_FIXED; in emulate_iret_real()
2149 ctxt->ops->set_nmi_mask(ctxt, false); in emulate_iret_real()
2156 switch(ctxt->mode) { in em_iret()
2164 /* iret from protected mode unimplemented yet */ in em_iret()
2174 u8 cpl = ctxt->ops->cpl(ctxt); in em_jmp_far()
2176 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2); in em_jmp_far()
2184 rc = assign_eip_far(ctxt, ctxt->src.val); in em_jmp_far()
2194 return assign_eip_near(ctxt, ctxt->src.val); in em_jmp_abs()
2202 old_eip = ctxt->_eip; in em_call_near_abs()
2203 rc = assign_eip_near(ctxt, ctxt->src.val); in em_call_near_abs()
2206 ctxt->src.val = old_eip; in em_call_near_abs()
2213 u64 old = ctxt->dst.orig_val64; in em_cmpxchg8b()
2215 if (ctxt->dst.bytes == 16) in em_cmpxchg8b()
2222 ctxt->eflags &= ~X86_EFLAGS_ZF; in em_cmpxchg8b()
2224 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) | in em_cmpxchg8b()
2227 ctxt->eflags |= X86_EFLAGS_ZF; in em_cmpxchg8b()
2237 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes); in em_ret()
2249 int cpl = ctxt->ops->cpl(ctxt); in em_ret_far()
2252 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes); in em_ret_far()
2255 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes); in em_ret_far()
2278 rsp_increment(ctxt, ctxt->src.val); in em_ret_far_imm()
2285 ctxt->dst.orig_val = ctxt->dst.val; in em_cmpxchg()
2286 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX); in em_cmpxchg()
2287 ctxt->src.orig_val = ctxt->src.val; in em_cmpxchg()
2288 ctxt->src.val = ctxt->dst.orig_val; in em_cmpxchg()
2291 if (ctxt->eflags & X86_EFLAGS_ZF) { in em_cmpxchg()
2293 ctxt->src.type = OP_NONE; in em_cmpxchg()
2294 ctxt->dst.val = ctxt->src.orig_val; in em_cmpxchg()
2297 ctxt->src.type = OP_REG; in em_cmpxchg()
2298 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX); in em_cmpxchg()
2299 ctxt->src.val = ctxt->dst.orig_val; in em_cmpxchg()
2300 /* Create write-cycle to dest by writing the same value */ in em_cmpxchg()
2301 ctxt->dst.val = ctxt->dst.orig_val; in em_cmpxchg()
2308 int seg = ctxt->src2.val; in em_lseg()
2312 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2); in em_lseg()
2318 ctxt->dst.val = ctxt->src.val; in em_lseg()
2324 if (!ctxt->ops->is_smm(ctxt)) in em_rsm()
2327 if (ctxt->ops->leave_smm(ctxt)) in em_rsm()
2328 ctxt->ops->triple_fault(ctxt); in em_rsm()
2336 cs->l = 0; /* will be adjusted later */ in setup_syscalls_segments()
2338 cs->g = 1; /* 4kb granularity */ in setup_syscalls_segments()
2340 cs->type = 0x0b; /* Read, Execute, Accessed */ in setup_syscalls_segments()
2341 cs->s = 1; in setup_syscalls_segments()
2342 cs->dpl = 0; /* will be adjusted later */ in setup_syscalls_segments()
2343 cs->p = 1; in setup_syscalls_segments()
2344 cs->d = 1; in setup_syscalls_segments()
2345 cs->avl = 0; in setup_syscalls_segments()
2349 ss->g = 1; /* 4kb granularity */ in setup_syscalls_segments()
2350 ss->s = 1; in setup_syscalls_segments()
2351 ss->type = 0x03; /* Read/Write, Accessed */ in setup_syscalls_segments()
2352 ss->d = 1; /* 32bit stack segment */ in setup_syscalls_segments()
2353 ss->dpl = 0; in setup_syscalls_segments()
2354 ss->p = 1; in setup_syscalls_segments()
2355 ss->l = 0; in setup_syscalls_segments()
2356 ss->avl = 0; in setup_syscalls_segments()
2361 const struct x86_emulate_ops *ops = ctxt->ops; in em_syscall()
2367 /* syscall is not available in real mode */ in em_syscall()
2368 if (ctxt->mode == X86EMUL_MODE_REAL || in em_syscall()
2369 ctxt->mode == X86EMUL_MODE_VM86) in em_syscall()
2373 * Intel compatible CPUs only support SYSCALL in 64-bit mode, whereas in em_syscall()
2374 * AMD allows SYSCALL in any flavor of protected mode. Note, it's in em_syscall()
2376 * as SYSCALL won't fault in the "wrong" mode, i.e. there is no #UD in em_syscall()
2377 * for KVM to trap-and-emulate, unlike emulating AMD on Intel. in em_syscall()
2379 if (ctxt->mode != X86EMUL_MODE_PROT64 && in em_syscall()
2380 ctxt->ops->guest_cpuid_is_intel_compatible(ctxt)) in em_syscall()
2383 ops->get_msr(ctxt, MSR_EFER, &efer); in em_syscall()
2388 ops->get_msr(ctxt, MSR_STAR, &msr_data); in em_syscall()
2397 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS); in em_syscall()
2398 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS); in em_syscall()
2400 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip; in em_syscall()
2403 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags; in em_syscall()
2405 ops->get_msr(ctxt, in em_syscall()
2406 ctxt->mode == X86EMUL_MODE_PROT64 ? in em_syscall()
2408 ctxt->_eip = msr_data; in em_syscall()
2410 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data); in em_syscall()
2411 ctxt->eflags &= ~msr_data; in em_syscall()
2412 ctxt->eflags |= X86_EFLAGS_FIXED; in em_syscall()
2415 /* legacy mode */ in em_syscall()
2416 ops->get_msr(ctxt, MSR_STAR, &msr_data); in em_syscall()
2417 ctxt->_eip = (u32)msr_data; in em_syscall()
2419 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF); in em_syscall()
2422 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0; in em_syscall()
2428 const struct x86_emulate_ops *ops = ctxt->ops; in em_sysenter()
2434 ops->get_msr(ctxt, MSR_EFER, &efer); in em_sysenter()
2435 /* inject #GP if in real mode */ in em_sysenter()
2436 if (ctxt->mode == X86EMUL_MODE_REAL) in em_sysenter()
2440 * Intel's architecture allows SYSENTER in compatibility mode, but AMD in em_sysenter()
2441 * does not. Note, AMD does allow SYSENTER in legacy protected mode. in em_sysenter()
2443 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA) && in em_sysenter()
2444 !ctxt->ops->guest_cpuid_is_intel_compatible(ctxt)) in em_sysenter()
2447 /* sysenter/sysexit have not been tested in 64bit mode. */ in em_sysenter()
2448 if (ctxt->mode == X86EMUL_MODE_PROT64) in em_sysenter()
2451 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data); in em_sysenter()
2456 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF); in em_sysenter()
2464 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS); in em_sysenter()
2465 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS); in em_sysenter()
2467 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data); in em_sysenter()
2468 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data; in em_sysenter()
2470 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data); in em_sysenter()
2474 ctxt->mode = X86EMUL_MODE_PROT64; in em_sysenter()
2481 const struct x86_emulate_ops *ops = ctxt->ops; in em_sysexit()
2487 /* inject #GP if in real mode or Virtual 8086 mode */ in em_sysexit()
2488 if (ctxt->mode == X86EMUL_MODE_REAL || in em_sysexit()
2489 ctxt->mode == X86EMUL_MODE_VM86) in em_sysexit()
2494 if ((ctxt->rex_prefix & 0x8) != 0x0) in em_sysexit()
2504 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data); in em_sysexit()
2529 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS); in em_sysexit()
2530 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS); in em_sysexit()
2532 ctxt->_eip = rdx; in em_sysexit()
2533 ctxt->mode = usermode; in em_sysexit()
2542 if (ctxt->mode == X86EMUL_MODE_REAL) in emulator_bad_iopl()
2544 if (ctxt->mode == X86EMUL_MODE_VM86) in emulator_bad_iopl()
2546 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT; in emulator_bad_iopl()
2547 return ctxt->ops->cpl(ctxt) > iopl; in emulator_bad_iopl()
2556 const struct x86_emulate_ops *ops = ctxt->ops; in emulator_io_port_access_allowed()
2561 unsigned mask = (1 << len) - 1; in emulator_io_port_access_allowed()
2572 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR); in emulator_io_port_access_allowed()
2581 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true); in emulator_io_port_access_allowed()
2586 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true); in emulator_io_port_access_allowed()
2597 if (ctxt->perm_ok) in emulator_io_permitted()
2604 ctxt->perm_ok = true; in emulator_io_permitted()
2613 * manner when ECX is zero due to REP-string optimizations. in string_registers_quirk()
2618 if (ctxt->ad_bytes != 4) in string_registers_quirk()
2622 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true); in string_registers_quirk()
2628 switch (ctxt->b) { in string_registers_quirk()
2631 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1; in string_registers_quirk()
2635 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1; in string_registers_quirk()
2643 tss->ip = ctxt->_eip; in save_state_to_tss16()
2644 tss->flag = ctxt->eflags; in save_state_to_tss16()
2645 tss->ax = reg_read(ctxt, VCPU_REGS_RAX); in save_state_to_tss16()
2646 tss->cx = reg_read(ctxt, VCPU_REGS_RCX); in save_state_to_tss16()
2647 tss->dx = reg_read(ctxt, VCPU_REGS_RDX); in save_state_to_tss16()
2648 tss->bx = reg_read(ctxt, VCPU_REGS_RBX); in save_state_to_tss16()
2649 tss->sp = reg_read(ctxt, VCPU_REGS_RSP); in save_state_to_tss16()
2650 tss->bp = reg_read(ctxt, VCPU_REGS_RBP); in save_state_to_tss16()
2651 tss->si = reg_read(ctxt, VCPU_REGS_RSI); in save_state_to_tss16()
2652 tss->di = reg_read(ctxt, VCPU_REGS_RDI); in save_state_to_tss16()
2654 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES); in save_state_to_tss16()
2655 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS); in save_state_to_tss16()
2656 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS); in save_state_to_tss16()
2657 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS); in save_state_to_tss16()
2658 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR); in save_state_to_tss16()
2667 ctxt->_eip = tss->ip; in load_state_from_tss16()
2668 ctxt->eflags = tss->flag | 2; in load_state_from_tss16()
2669 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax; in load_state_from_tss16()
2670 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx; in load_state_from_tss16()
2671 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx; in load_state_from_tss16()
2672 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx; in load_state_from_tss16()
2673 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp; in load_state_from_tss16()
2674 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp; in load_state_from_tss16()
2675 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si; in load_state_from_tss16()
2676 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di; in load_state_from_tss16()
2682 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR); in load_state_from_tss16()
2683 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES); in load_state_from_tss16()
2684 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS); in load_state_from_tss16()
2685 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS); in load_state_from_tss16()
2686 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS); in load_state_from_tss16()
2688 cpl = tss->cs & 3; in load_state_from_tss16()
2694 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl, in load_state_from_tss16()
2698 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, in load_state_from_tss16()
2702 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, in load_state_from_tss16()
2706 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, in load_state_from_tss16()
2710 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, in load_state_from_tss16()
2756 tss->eip = ctxt->_eip; in save_state_to_tss32()
2757 tss->eflags = ctxt->eflags; in save_state_to_tss32()
2758 tss->eax = reg_read(ctxt, VCPU_REGS_RAX); in save_state_to_tss32()
2759 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX); in save_state_to_tss32()
2760 tss->edx = reg_read(ctxt, VCPU_REGS_RDX); in save_state_to_tss32()
2761 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX); in save_state_to_tss32()
2762 tss->esp = reg_read(ctxt, VCPU_REGS_RSP); in save_state_to_tss32()
2763 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP); in save_state_to_tss32()
2764 tss->esi = reg_read(ctxt, VCPU_REGS_RSI); in save_state_to_tss32()
2765 tss->edi = reg_read(ctxt, VCPU_REGS_RDI); in save_state_to_tss32()
2767 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES); in save_state_to_tss32()
2768 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS); in save_state_to_tss32()
2769 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS); in save_state_to_tss32()
2770 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS); in save_state_to_tss32()
2771 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS); in save_state_to_tss32()
2772 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS); in save_state_to_tss32()
2781 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3)) in load_state_from_tss32()
2783 ctxt->_eip = tss->eip; in load_state_from_tss32()
2784 ctxt->eflags = tss->eflags | 2; in load_state_from_tss32()
2787 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax; in load_state_from_tss32()
2788 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx; in load_state_from_tss32()
2789 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx; in load_state_from_tss32()
2790 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx; in load_state_from_tss32()
2791 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp; in load_state_from_tss32()
2792 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp; in load_state_from_tss32()
2793 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi; in load_state_from_tss32()
2794 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi; in load_state_from_tss32()
2801 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR); in load_state_from_tss32()
2802 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES); in load_state_from_tss32()
2803 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS); in load_state_from_tss32()
2804 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS); in load_state_from_tss32()
2805 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS); in load_state_from_tss32()
2806 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS); in load_state_from_tss32()
2807 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS); in load_state_from_tss32()
2810 * If we're switching between Protected Mode and VM86, we need to make in load_state_from_tss32()
2811 * sure to update the mode before loading the segment descriptors so in load_state_from_tss32()
2814 if (ctxt->eflags & X86_EFLAGS_VM) { in load_state_from_tss32()
2815 ctxt->mode = X86EMUL_MODE_VM86; in load_state_from_tss32()
2818 ctxt->mode = X86EMUL_MODE_PROT32; in load_state_from_tss32()
2819 cpl = tss->cs & 3; in load_state_from_tss32()
2826 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR, in load_state_from_tss32()
2830 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, in load_state_from_tss32()
2834 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, in load_state_from_tss32()
2838 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, in load_state_from_tss32()
2842 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, in load_state_from_tss32()
2846 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl, in load_state_from_tss32()
2850 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl, in load_state_from_tss32()
2873 ldt_sel_offset - eip_offset); in task_switch_32()
2898 const struct x86_emulate_ops *ops = ctxt->ops; in emulator_do_task_switch()
2903 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR); in emulator_do_task_switch()
2923 * 3. jmp/call to TSS/task-gate: No check is performed since the in emulator_do_task_switch()
2927 if (idt_index != -1) { in emulator_do_task_switch()
2938 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl) in emulator_do_task_switch()
2956 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT; in emulator_do_task_switch()
2972 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT; in emulator_do_task_switch()
2979 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS); in emulator_do_task_switch()
2980 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR); in emulator_do_task_switch()
2983 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2; in emulator_do_task_switch()
2984 ctxt->lock_prefix = 0; in emulator_do_task_switch()
2985 ctxt->src.val = (unsigned long) error_code; in emulator_do_task_switch()
2989 dr7 = ops->get_dr(ctxt, 7); in emulator_do_task_switch()
2990 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN)); in emulator_do_task_switch()
3002 ctxt->_eip = ctxt->eip; in emulator_task_switch()
3003 ctxt->dst.type = OP_NONE; in emulator_task_switch()
3009 ctxt->eip = ctxt->_eip; in emulator_task_switch()
3017 struct operand *op) in string_addr_inc() argument
3019 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count; in string_addr_inc()
3021 register_address_increment(ctxt, reg, df * op->bytes); in string_addr_inc()
3022 op->addr.mem.ea = register_address(ctxt, reg); in string_addr_inc()
3030 cf = ctxt->eflags & X86_EFLAGS_CF; in em_das()
3031 al = ctxt->dst.val; in em_das()
3036 af = ctxt->eflags & X86_EFLAGS_AF; in em_das()
3038 al -= 6; in em_das()
3045 al -= 0x60; in em_das()
3049 ctxt->dst.val = al; in em_das()
3051 ctxt->src.type = OP_IMM; in em_das()
3052 ctxt->src.val = 0; in em_das()
3053 ctxt->src.bytes = 1; in em_das()
3055 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF); in em_das()
3057 ctxt->eflags |= X86_EFLAGS_CF; in em_das()
3059 ctxt->eflags |= X86_EFLAGS_AF; in em_das()
3067 if (ctxt->src.val == 0) in em_aam()
3070 al = ctxt->dst.val & 0xff; in em_aam()
3071 ah = al / ctxt->src.val; in em_aam()
3072 al %= ctxt->src.val; in em_aam()
3074 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8); in em_aam()
3077 ctxt->src.type = OP_IMM; in em_aam()
3078 ctxt->src.val = 0; in em_aam()
3079 ctxt->src.bytes = 1; in em_aam()
3087 u8 al = ctxt->dst.val & 0xff; in em_aad()
3088 u8 ah = (ctxt->dst.val >> 8) & 0xff; in em_aad()
3090 al = (al + (ah * ctxt->src.val)) & 0xff; in em_aad()
3092 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al; in em_aad()
3095 ctxt->src.type = OP_IMM; in em_aad()
3096 ctxt->src.val = 0; in em_aad()
3097 ctxt->src.bytes = 1; in em_aad()
3106 long rel = ctxt->src.val; in em_call()
3108 ctxt->src.val = (unsigned long)ctxt->_eip; in em_call()
3121 const struct x86_emulate_ops *ops = ctxt->ops; in em_call_far()
3122 int cpl = ctxt->ops->cpl(ctxt); in em_call_far()
3123 enum x86emul_mode prev_mode = ctxt->mode; in em_call_far()
3125 old_eip = ctxt->_eip; in em_call_far()
3126 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS); in em_call_far()
3128 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2); in em_call_far()
3134 rc = assign_eip_far(ctxt, ctxt->src.val); in em_call_far()
3138 ctxt->src.val = old_cs; in em_call_far()
3143 ctxt->src.val = old_eip; in em_call_far()
3153 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS); in em_call_far()
3154 ctxt->mode = prev_mode; in em_call_far()
3164 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes); in em_ret_near_imm()
3170 rsp_increment(ctxt, ctxt->src.val); in em_ret_near_imm()
3177 ctxt->src.val = ctxt->dst.val; in em_xchg()
3178 write_register_operand(&ctxt->src); in em_xchg()
3181 ctxt->dst.val = ctxt->src.orig_val; in em_xchg()
3182 ctxt->lock_prefix = 1; in em_xchg()
3188 ctxt->dst.val = ctxt->src2.val; in em_imul_3op()
3194 ctxt->dst.type = OP_REG; in em_cwd()
3195 ctxt->dst.bytes = ctxt->src.bytes; in em_cwd()
3196 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX); in em_cwd()
3197 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1); in em_cwd()
3206 if (!ctxt->ops->guest_has_rdpid(ctxt)) in em_rdpid()
3209 ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux); in em_rdpid()
3210 ctxt->dst.val = tsc_aux; in em_rdpid()
3218 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc); in em_rdtsc()
3228 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc)) in em_rdpmc()
3237 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr)); in em_mov()
3245 if (!ctxt->ops->guest_has_movbe(ctxt)) in em_movbe()
3248 switch (ctxt->op_bytes) { in em_movbe()
3255 * Both casting ->valptr and ->val to u16 breaks strict aliasing in em_movbe()
3258 tmp = (u16)ctxt->src.val; in em_movbe()
3259 ctxt->dst.val &= ~0xffffUL; in em_movbe()
3260 ctxt->dst.val |= (unsigned long)swab16(tmp); in em_movbe()
3263 ctxt->dst.val = swab32((u32)ctxt->src.val); in em_movbe()
3266 ctxt->dst.val = swab64(ctxt->src.val); in em_movbe()
3276 int cr_num = ctxt->modrm_reg; in em_cr_write()
3279 if (ctxt->ops->set_cr(ctxt, cr_num, ctxt->src.val)) in em_cr_write()
3283 ctxt->dst.type = OP_NONE; in em_cr_write()
3288 * which can affect the cpu's execution mode. in em_cr_write()
3302 if (ctxt->mode == X86EMUL_MODE_PROT64) in em_dr_write()
3303 val = ctxt->src.val & ~0ULL; in em_dr_write()
3305 val = ctxt->src.val & ~0U; in em_dr_write()
3308 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0) in em_dr_write()
3312 ctxt->dst.type = OP_NONE; in em_dr_write()
3324 r = ctxt->ops->set_msr_with_filter(ctxt, msr_index, msr_data); in em_wrmsr()
3338 r = ctxt->ops->get_msr_with_filter(ctxt, msr_index, &msr_data); in em_rdmsr()
3353 (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) && in em_store_sreg()
3354 ctxt->ops->cpl(ctxt) > 0) in em_store_sreg()
3357 ctxt->dst.val = get_segment_selector(ctxt, segment); in em_store_sreg()
3358 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM) in em_store_sreg()
3359 ctxt->dst.bytes = 2; in em_store_sreg()
3365 if (ctxt->modrm_reg > VCPU_SREG_GS) in em_mov_rm_sreg()
3368 return em_store_sreg(ctxt, ctxt->modrm_reg); in em_mov_rm_sreg()
3373 u16 sel = ctxt->src.val; in em_mov_sreg_rm()
3375 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS) in em_mov_sreg_rm()
3378 if (ctxt->modrm_reg == VCPU_SREG_SS) in em_mov_sreg_rm()
3379 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS; in em_mov_sreg_rm()
3382 ctxt->dst.type = OP_NONE; in em_mov_sreg_rm()
3383 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg); in em_mov_sreg_rm()
3393 u16 sel = ctxt->src.val; in em_lldt()
3396 ctxt->dst.type = OP_NONE; in em_lldt()
3407 u16 sel = ctxt->src.val; in em_ltr()
3410 ctxt->dst.type = OP_NONE; in em_ltr()
3420 rc = __linearize(ctxt, ctxt->src.addr.mem, &max_size, 1, ctxt->mode, in em_invlpg()
3423 ctxt->ops->invlpg(ctxt, linear); in em_invlpg()
3425 ctxt->dst.type = OP_NONE; in em_invlpg()
3433 cr0 = ctxt->ops->get_cr(ctxt, 0); in em_clts()
3435 ctxt->ops->set_cr(ctxt, 0, cr0); in em_clts()
3441 int rc = ctxt->ops->fix_hypercall(ctxt); in em_hypercall()
3446 /* Let the processor re-execute the fixed hypercall */ in em_hypercall()
3447 ctxt->_eip = ctxt->eip; in em_hypercall()
3449 ctxt->dst.type = OP_NONE; in em_hypercall()
3459 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) && in emulate_store_desc_ptr()
3460 ctxt->ops->cpl(ctxt) > 0) in emulate_store_desc_ptr()
3463 if (ctxt->mode == X86EMUL_MODE_PROT64) in emulate_store_desc_ptr()
3464 ctxt->op_bytes = 8; in emulate_store_desc_ptr()
3466 if (ctxt->op_bytes == 2) { in emulate_store_desc_ptr()
3467 ctxt->op_bytes = 4; in emulate_store_desc_ptr()
3471 ctxt->dst.type = OP_NONE; in emulate_store_desc_ptr()
3472 return segmented_write_std(ctxt, ctxt->dst.addr.mem, in emulate_store_desc_ptr()
3473 &desc_ptr, 2 + ctxt->op_bytes); in emulate_store_desc_ptr()
3478 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt); in em_sgdt()
3483 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt); in em_sidt()
3491 if (ctxt->mode == X86EMUL_MODE_PROT64) in em_lgdt_lidt()
3492 ctxt->op_bytes = 8; in em_lgdt_lidt()
3493 rc = read_descriptor(ctxt, ctxt->src.addr.mem, in em_lgdt_lidt()
3495 ctxt->op_bytes); in em_lgdt_lidt()
3498 if (ctxt->mode == X86EMUL_MODE_PROT64 && in em_lgdt_lidt()
3503 ctxt->ops->set_gdt(ctxt, &desc_ptr); in em_lgdt_lidt()
3505 ctxt->ops->set_idt(ctxt, &desc_ptr); in em_lgdt_lidt()
3507 ctxt->dst.type = OP_NONE; in em_lgdt_lidt()
3523 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) && in em_smsw()
3524 ctxt->ops->cpl(ctxt) > 0) in em_smsw()
3527 if (ctxt->dst.type == OP_MEM) in em_smsw()
3528 ctxt->dst.bytes = 2; in em_smsw()
3529 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0); in em_smsw()
3535 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul) in em_lmsw()
3536 | (ctxt->src.val & 0x0f)); in em_lmsw()
3537 ctxt->dst.type = OP_NONE; in em_lmsw()
3545 register_address_increment(ctxt, VCPU_REGS_RCX, -1); in em_loop()
3547 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags))) in em_loop()
3548 rc = jmp_rel(ctxt, ctxt->src.val); in em_loop()
3558 rc = jmp_rel(ctxt, ctxt->src.val); in em_jcxz()
3565 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val, in em_in()
3566 &ctxt->dst.val)) in em_in()
3574 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val, in em_out()
3575 &ctxt->src.val, 1); in em_out()
3577 ctxt->dst.type = OP_NONE; in em_out()
3586 ctxt->eflags &= ~X86_EFLAGS_IF; in em_cli()
3595 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI; in em_sti()
3596 ctxt->eflags |= X86_EFLAGS_IF; in em_sti()
3605 ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr); in em_cpuid()
3607 ctxt->ops->cpl(ctxt)) { in em_cpuid()
3613 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false); in em_cpuid()
3629 ctxt->eflags &= ~0xffUL; in em_sahf()
3630 ctxt->eflags |= flags | X86_EFLAGS_FIXED; in em_sahf()
3637 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8; in em_lahf()
3643 switch (ctxt->op_bytes) { in em_bswap()
3646 asm("bswap %0" : "+r"(ctxt->dst.val)); in em_bswap()
3650 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val)); in em_bswap()
3670 ctxt->dst.val = (s32) ctxt->src.val; in em_movsxd()
3676 if (!ctxt->ops->guest_has_fxsr(ctxt)) in check_fxsr()
3679 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM)) in check_fxsr()
3686 if (ctxt->mode >= X86EMUL_MODE_PROT64) in check_fxsr()
3693 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but does save
3704 if (ctxt->mode == X86EMUL_MODE_PROT64) in fxstate_size()
3707 cr4_osfxsr = ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR; in fxstate_size()
3712 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
3713 * 1) 16 bit mode
3714 * 2) 32 bit mode
3715 * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs
3718 * 3) 64-bit mode with REX.W prefix
3719 * - like (2), but XMM 8-15 are being saved and restored
3720 * 4) 64-bit mode without REX.W prefix
3721 * - like (3), but FIP and FDP are 64 bit
3723 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
3747 return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state, in em_fxsave()
3766 __fxstate_size(16) - used_size); in fxregs_fixup()
3782 rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, size); in em_fxrstor()
3812 if (!(ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSXSAVE)) in em_xsetbv()
3819 if (ctxt->ops->set_xcr(ctxt, ecx, ((u64)edx << 32) | eax)) in em_xsetbv()
3839 if (!valid_cr(ctxt->modrm_reg)) in check_cr_access()
3847 int dr = ctxt->modrm_reg; in check_dr_read()
3853 cr4 = ctxt->ops->get_cr(ctxt, 4); in check_dr_read()
3857 if (ctxt->ops->get_dr(ctxt, 7) & DR7_GD) { in check_dr_read()
3860 dr6 = ctxt->ops->get_dr(ctxt, 6); in check_dr_read()
3863 ctxt->ops->set_dr(ctxt, 6, dr6); in check_dr_read()
3872 u64 new_val = ctxt->src.val64; in check_dr_write()
3873 int dr = ctxt->modrm_reg; in check_dr_write()
3885 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer); in check_svme()
3906 u64 cr4 = ctxt->ops->get_cr(ctxt, 4); in check_rdtsc()
3908 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt)) in check_rdtsc()
3916 u64 cr4 = ctxt->ops->get_cr(ctxt, 4); in check_rdpmc()
3920 * VMware allows access to these Pseduo-PMCs even when read via RDPMC in check_rdpmc()
3929 * protected mode. in check_rdpmc()
3931 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) || in check_rdpmc()
3932 ctxt->ops->check_rdpmc_early(ctxt, rcx)) in check_rdpmc()
3940 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u); in check_perm_in()
3941 if (!emulator_io_permitted(ctxt, ctxt->src.val, ctxt->dst.bytes)) in check_perm_in()
3949 ctxt->src.bytes = min(ctxt->src.bytes, 4u); in check_perm_out()
3950 if (!emulator_io_permitted(ctxt, ctxt->dst.val, ctxt->src.bytes)) in check_perm_out()
4169 /* 0xC0 - 0xC7 */
4171 /* 0xC8 - 0xCF */
4173 /* 0xD0 - 0xC7 */
4175 /* 0xD8 - 0xDF */
4177 /* 0xE0 - 0xE7 */
4179 /* 0xE8 - 0xEF */
4181 /* 0xF0 - 0xF7 */
4183 /* 0xF8 - 0xFF */
4190 /* 0xC0 - 0xC7 */
4192 /* 0xC8 - 0xCF */
4194 /* 0xD0 - 0xC7 */
4196 /* 0xD8 - 0xDF */
4198 /* 0xE0 - 0xE7 */
4200 /* 0xE8 - 0xEF */
4202 /* 0xF0 - 0xF7 */
4204 /* 0xF8 - 0xFF */
4211 /* 0xC0 - 0xC7 */
4213 /* 0xC8 - 0xCF */
4215 /* 0xD0 - 0xC7 */
4217 /* 0xD8 - 0xDF */
4219 /* 0xE0 - 0xE7 */
4221 /* 0xE8 - 0xEF */
4223 /* 0xF0 - 0xF7 */
4225 /* 0xF8 - 0xFF */
4242 /* 0x00 - 0x07 */
4246 /* 0x08 - 0x0F */
4250 /* 0x10 - 0x17 */
4254 /* 0x18 - 0x1F */
4258 /* 0x20 - 0x27 */
4260 /* 0x28 - 0x2F */
4262 /* 0x30 - 0x37 */
4264 /* 0x38 - 0x3F */
4266 /* 0x40 - 0x4F */
4268 /* 0x50 - 0x57 */
4270 /* 0x58 - 0x5F */
4272 /* 0x60 - 0x67 */
4277 /* 0x68 - 0x6F */
4284 /* 0x70 - 0x7F */
4286 /* 0x80 - 0x87 */
4293 /* 0x88 - 0x8F */
4300 /* 0x90 - 0x97 */
4302 /* 0x98 - 0x9F */
4308 /* 0xA0 - 0xA7 */
4313 /* 0xA8 - 0xAF */
4318 /* 0xB0 - 0xB7 */
4320 /* 0xB8 - 0xBF */
4322 /* 0xC0 - 0xC7 */
4329 /* 0xC8 - 0xCF */
4337 /* 0xD0 - 0xD7 */
4344 /* 0xD8 - 0xDF */
4346 /* 0xE0 - 0xE7 */
4351 /* 0xE8 - 0xEF */
4358 /* 0xF0 - 0xF7 */
4362 /* 0xF8 - 0xFF */
4369 /* 0x00 - 0x0F */
4375 /* 0x10 - 0x1F */
4385 /* 0x20 - 0x2F */
4397 /* 0x30 - 0x3F */
4406 /* 0x40 - 0x4F */
4408 /* 0x50 - 0x5F */
4410 /* 0x60 - 0x6F */
4415 /* 0x70 - 0x7F */
4420 /* 0x80 - 0x8F */
4422 /* 0x90 - 0x9F */
4424 /* 0xA0 - 0xA7 */
4430 /* 0xA8 - 0xAF */
4437 /* 0xB0 - 0xB7 */
4444 /* 0xB8 - 0xBF */
4451 /* 0xC0 - 0xC7 */
4455 /* 0xC8 - 0xCF */
4457 /* 0xD0 - 0xDF */
4459 /* 0xE0 - 0xEF */
4462 /* 0xF0 - 0xFF */
4487 /* 0x00 - 0x7f */
4489 /* 0x80 - 0xef */
4491 /* 0xf0 - 0xf1 */
4494 /* 0xf2 - 0xff */
4518 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; in imm_size()
4524 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op, in decode_imm() argument
4529 op->type = OP_IMM; in decode_imm()
4530 op->bytes = size; in decode_imm()
4531 op->addr.mem.ea = ctxt->_eip; in decode_imm()
4532 /* NB. Immediates are sign-extended as necessary. */ in decode_imm()
4533 switch (op->bytes) { in decode_imm()
4535 op->val = insn_fetch(s8, ctxt); in decode_imm()
4538 op->val = insn_fetch(s16, ctxt); in decode_imm()
4541 op->val = insn_fetch(s32, ctxt); in decode_imm()
4544 op->val = insn_fetch(s64, ctxt); in decode_imm()
4548 switch (op->bytes) { in decode_imm()
4550 op->val &= 0xff; in decode_imm()
4553 op->val &= 0xffff; in decode_imm()
4556 op->val &= 0xffffffff; in decode_imm()
4564 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op, in decode_operand() argument
4571 decode_register_operand(ctxt, op); in decode_operand()
4574 rc = decode_imm(ctxt, op, 1, false); in decode_operand()
4577 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; in decode_operand()
4579 *op = ctxt->memop; in decode_operand()
4580 ctxt->memopp = op; in decode_operand()
4581 if (ctxt->d & BitOp) in decode_operand()
4583 op->orig_val = op->val; in decode_operand()
4586 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8; in decode_operand()
4589 op->type = OP_REG; in decode_operand()
4590 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; in decode_operand()
4591 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX); in decode_operand()
4592 fetch_register_operand(op); in decode_operand()
4593 op->orig_val = op->val; in decode_operand()
4596 op->type = OP_REG; in decode_operand()
4597 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes; in decode_operand()
4598 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX); in decode_operand()
4599 fetch_register_operand(op); in decode_operand()
4600 op->orig_val = op->val; in decode_operand()
4603 if (ctxt->d & ByteOp) { in decode_operand()
4604 op->type = OP_NONE; in decode_operand()
4607 op->type = OP_REG; in decode_operand()
4608 op->bytes = ctxt->op_bytes; in decode_operand()
4609 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX); in decode_operand()
4610 fetch_register_operand(op); in decode_operand()
4611 op->orig_val = op->val; in decode_operand()
4614 op->type = OP_MEM; in decode_operand()
4615 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; in decode_operand()
4616 op->addr.mem.ea = in decode_operand()
4618 op->addr.mem.seg = VCPU_SREG_ES; in decode_operand()
4619 op->val = 0; in decode_operand()
4620 op->count = 1; in decode_operand()
4623 op->type = OP_REG; in decode_operand()
4624 op->bytes = 2; in decode_operand()
4625 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX); in decode_operand()
4626 fetch_register_operand(op); in decode_operand()
4629 op->type = OP_IMM; in decode_operand()
4630 op->bytes = 1; in decode_operand()
4631 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff; in decode_operand()
4634 rc = decode_imm(ctxt, op, 1, true); in decode_operand()
4637 op->type = OP_IMM; in decode_operand()
4638 op->bytes = 1; in decode_operand()
4639 op->val = 1; in decode_operand()
4642 rc = decode_imm(ctxt, op, imm_size(ctxt), true); in decode_operand()
4645 rc = decode_imm(ctxt, op, ctxt->op_bytes, true); in decode_operand()
4648 ctxt->memop.bytes = 1; in decode_operand()
4649 if (ctxt->memop.type == OP_REG) { in decode_operand()
4650 ctxt->memop.addr.reg = decode_register(ctxt, in decode_operand()
4651 ctxt->modrm_rm, true); in decode_operand()
4652 fetch_register_operand(&ctxt->memop); in decode_operand()
4656 ctxt->memop.bytes = 2; in decode_operand()
4659 ctxt->memop.bytes = 4; in decode_operand()
4662 rc = decode_imm(ctxt, op, 2, false); in decode_operand()
4665 rc = decode_imm(ctxt, op, imm_size(ctxt), false); in decode_operand()
4668 op->type = OP_MEM; in decode_operand()
4669 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; in decode_operand()
4670 op->addr.mem.ea = in decode_operand()
4672 op->addr.mem.seg = ctxt->seg_override; in decode_operand()
4673 op->val = 0; in decode_operand()
4674 op->count = 1; in decode_operand()
4677 op->type = OP_MEM; in decode_operand()
4678 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; in decode_operand()
4679 op->addr.mem.ea = in decode_operand()
4683 op->addr.mem.seg = ctxt->seg_override; in decode_operand()
4684 op->val = 0; in decode_operand()
4687 op->type = OP_IMM; in decode_operand()
4688 op->addr.mem.ea = ctxt->_eip; in decode_operand()
4689 op->bytes = ctxt->op_bytes + 2; in decode_operand()
4690 insn_fetch_arr(op->valptr, op->bytes, ctxt); in decode_operand()
4693 ctxt->memop.bytes = ctxt->op_bytes + 2; in decode_operand()
4696 op->type = OP_IMM; in decode_operand()
4697 op->val = VCPU_SREG_ES; in decode_operand()
4700 op->type = OP_IMM; in decode_operand()
4701 op->val = VCPU_SREG_CS; in decode_operand()
4704 op->type = OP_IMM; in decode_operand()
4705 op->val = VCPU_SREG_SS; in decode_operand()
4708 op->type = OP_IMM; in decode_operand()
4709 op->val = VCPU_SREG_DS; in decode_operand()
4712 op->type = OP_IMM; in decode_operand()
4713 op->val = VCPU_SREG_FS; in decode_operand()
4716 op->type = OP_IMM; in decode_operand()
4717 op->val = VCPU_SREG_GS; in decode_operand()
4722 op->type = OP_NONE; /* Disable writeback. */ in decode_operand()
4733 int mode = ctxt->mode; in x86_decode_insn() local
4741 ctxt->memop.type = OP_NONE; in x86_decode_insn()
4742 ctxt->memopp = NULL; in x86_decode_insn()
4743 ctxt->_eip = ctxt->eip; in x86_decode_insn()
4744 ctxt->fetch.ptr = ctxt->fetch.data; in x86_decode_insn()
4745 ctxt->fetch.end = ctxt->fetch.data + insn_len; in x86_decode_insn()
4746 ctxt->opcode_len = 1; in x86_decode_insn()
4747 ctxt->intercept = x86_intercept_none; in x86_decode_insn()
4749 memcpy(ctxt->fetch.data, insn, insn_len); in x86_decode_insn()
4756 switch (mode) { in x86_decode_insn()
4760 ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS); in x86_decode_insn()
4780 ctxt->op_bytes = def_op_bytes; in x86_decode_insn()
4781 ctxt->ad_bytes = def_ad_bytes; in x86_decode_insn()
4785 switch (ctxt->b = insn_fetch(u8, ctxt)) { in x86_decode_insn()
4786 case 0x66: /* operand-size override */ in x86_decode_insn()
4789 ctxt->op_bytes = def_op_bytes ^ 6; in x86_decode_insn()
4791 case 0x67: /* address-size override */ in x86_decode_insn()
4792 if (mode == X86EMUL_MODE_PROT64) in x86_decode_insn()
4794 ctxt->ad_bytes = def_ad_bytes ^ 12; in x86_decode_insn()
4797 ctxt->ad_bytes = def_ad_bytes ^ 6; in x86_decode_insn()
4801 ctxt->seg_override = VCPU_SREG_ES; in x86_decode_insn()
4805 ctxt->seg_override = VCPU_SREG_CS; in x86_decode_insn()
4809 ctxt->seg_override = VCPU_SREG_SS; in x86_decode_insn()
4813 ctxt->seg_override = VCPU_SREG_DS; in x86_decode_insn()
4817 ctxt->seg_override = VCPU_SREG_FS; in x86_decode_insn()
4821 ctxt->seg_override = VCPU_SREG_GS; in x86_decode_insn()
4824 if (mode != X86EMUL_MODE_PROT64) in x86_decode_insn()
4826 ctxt->rex_prefix = ctxt->b; in x86_decode_insn()
4829 ctxt->lock_prefix = 1; in x86_decode_insn()
4833 ctxt->rep_prefix = ctxt->b; in x86_decode_insn()
4841 ctxt->rex_prefix = 0; in x86_decode_insn()
4847 if (ctxt->rex_prefix & 8) in x86_decode_insn()
4848 ctxt->op_bytes = 8; /* REX.W */ in x86_decode_insn()
4851 opcode = opcode_table[ctxt->b]; in x86_decode_insn()
4852 /* Two-byte opcode? */ in x86_decode_insn()
4853 if (ctxt->b == 0x0f) { in x86_decode_insn()
4854 ctxt->opcode_len = 2; in x86_decode_insn()
4855 ctxt->b = insn_fetch(u8, ctxt); in x86_decode_insn()
4856 opcode = twobyte_table[ctxt->b]; in x86_decode_insn()
4859 if (ctxt->b == 0x38) { in x86_decode_insn()
4860 ctxt->opcode_len = 3; in x86_decode_insn()
4861 ctxt->b = insn_fetch(u8, ctxt); in x86_decode_insn()
4862 opcode = opcode_map_0f_38[ctxt->b]; in x86_decode_insn()
4865 ctxt->d = opcode.flags; in x86_decode_insn()
4867 if (ctxt->d & ModRM) in x86_decode_insn()
4868 ctxt->modrm = insn_fetch(u8, ctxt); in x86_decode_insn()
4870 /* vex-prefix instructions are not implemented */ in x86_decode_insn()
4871 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) && in x86_decode_insn()
4872 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) { in x86_decode_insn()
4873 ctxt->d = NotImpl; in x86_decode_insn()
4876 while (ctxt->d & GroupMask) { in x86_decode_insn()
4877 switch (ctxt->d & GroupMask) { in x86_decode_insn()
4879 goffset = (ctxt->modrm >> 3) & 7; in x86_decode_insn()
4883 goffset = (ctxt->modrm >> 3) & 7; in x86_decode_insn()
4884 if ((ctxt->modrm >> 6) == 3) in x86_decode_insn()
4885 opcode = opcode.u.gdual->mod3[goffset]; in x86_decode_insn()
4887 opcode = opcode.u.gdual->mod012[goffset]; in x86_decode_insn()
4890 goffset = ctxt->modrm & 7; in x86_decode_insn()
4894 if (ctxt->rep_prefix && op_prefix) in x86_decode_insn()
4896 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix; in x86_decode_insn()
4898 case 0x00: opcode = opcode.u.gprefix->pfx_no; break; in x86_decode_insn()
4899 case 0x66: opcode = opcode.u.gprefix->pfx_66; break; in x86_decode_insn()
4900 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break; in x86_decode_insn()
4901 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break; in x86_decode_insn()
4905 if (ctxt->modrm > 0xbf) { in x86_decode_insn()
4906 size_t size = ARRAY_SIZE(opcode.u.esc->high); in x86_decode_insn()
4908 ctxt->modrm - 0xc0, size); in x86_decode_insn()
4910 opcode = opcode.u.esc->high[index]; in x86_decode_insn()
4912 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7]; in x86_decode_insn()
4916 if ((ctxt->modrm >> 6) == 3) in x86_decode_insn()
4917 opcode = opcode.u.idual->mod3; in x86_decode_insn()
4919 opcode = opcode.u.idual->mod012; in x86_decode_insn()
4922 if (ctxt->mode == X86EMUL_MODE_PROT64) in x86_decode_insn()
4923 opcode = opcode.u.mdual->mode64; in x86_decode_insn()
4925 opcode = opcode.u.mdual->mode32; in x86_decode_insn()
4931 ctxt->d &= ~(u64)GroupMask; in x86_decode_insn()
4932 ctxt->d |= opcode.flags; in x86_decode_insn()
4935 ctxt->is_branch = opcode.flags & IsBranch; in x86_decode_insn()
4938 if (ctxt->d == 0) in x86_decode_insn()
4941 ctxt->execute = opcode.u.execute; in x86_decode_insn()
4944 likely(!(ctxt->d & EmulateOnUD))) in x86_decode_insn()
4947 if (unlikely(ctxt->d & in x86_decode_insn()
4954 ctxt->check_perm = opcode.check_perm; in x86_decode_insn()
4955 ctxt->intercept = opcode.intercept; in x86_decode_insn()
4957 if (ctxt->d & NotImpl) in x86_decode_insn()
4960 if (mode == X86EMUL_MODE_PROT64) { in x86_decode_insn()
4961 if (ctxt->op_bytes == 4 && (ctxt->d & Stack)) in x86_decode_insn()
4962 ctxt->op_bytes = 8; in x86_decode_insn()
4963 else if (ctxt->d & NearBranch) in x86_decode_insn()
4964 ctxt->op_bytes = 8; in x86_decode_insn()
4967 if (ctxt->d & Op3264) { in x86_decode_insn()
4968 if (mode == X86EMUL_MODE_PROT64) in x86_decode_insn()
4969 ctxt->op_bytes = 8; in x86_decode_insn()
4971 ctxt->op_bytes = 4; in x86_decode_insn()
4974 if ((ctxt->d & No16) && ctxt->op_bytes == 2) in x86_decode_insn()
4975 ctxt->op_bytes = 4; in x86_decode_insn()
4977 if (ctxt->d & Sse) in x86_decode_insn()
4978 ctxt->op_bytes = 16; in x86_decode_insn()
4979 else if (ctxt->d & Mmx) in x86_decode_insn()
4980 ctxt->op_bytes = 8; in x86_decode_insn()
4984 if (ctxt->d & ModRM) { in x86_decode_insn()
4985 rc = decode_modrm(ctxt, &ctxt->memop); in x86_decode_insn()
4988 ctxt->seg_override = ctxt->modrm_seg; in x86_decode_insn()
4990 } else if (ctxt->d & MemAbs) in x86_decode_insn()
4991 rc = decode_abs(ctxt, &ctxt->memop); in x86_decode_insn()
4996 ctxt->seg_override = VCPU_SREG_DS; in x86_decode_insn()
4998 ctxt->memop.addr.mem.seg = ctxt->seg_override; in x86_decode_insn()
5004 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask); in x86_decode_insn()
5012 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask); in x86_decode_insn()
5017 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask); in x86_decode_insn()
5019 if (ctxt->rip_relative && likely(ctxt->memopp)) in x86_decode_insn()
5020 ctxt->memopp->addr.mem.ea = address_mask(ctxt, in x86_decode_insn()
5021 ctxt->memopp->addr.mem.ea + ctxt->_eip); in x86_decode_insn()
5025 ctxt->have_exception = true; in x86_decode_insn()
5031 return ctxt->d & PageTable; in x86_page_table_writing_insn()
5040 * - if REPE/REPZ and ZF = 0 then done in string_insn_completed()
5041 * - if REPNE/REPNZ and ZF = 1 then done in string_insn_completed()
5043 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) || in string_insn_completed()
5044 (ctxt->b == 0xae) || (ctxt->b == 0xaf)) in string_insn_completed()
5045 && (((ctxt->rep_prefix == REPE_PREFIX) && in string_insn_completed()
5046 ((ctxt->eflags & X86_EFLAGS_ZF) == 0)) in string_insn_completed()
5047 || ((ctxt->rep_prefix == REPNE_PREFIX) && in string_insn_completed()
5048 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF)))) in string_insn_completed()
5068 static void fetch_possible_mmx_operand(struct operand *op) in fetch_possible_mmx_operand() argument
5070 if (op->type == OP_MM) in fetch_possible_mmx_operand()
5071 kvm_read_mmx_reg(op->addr.mm, &op->mm_val); in fetch_possible_mmx_operand()
5076 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF; in fastop()
5078 if (!(ctxt->d & ByteOp)) in fastop()
5079 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE; in fastop()
5082 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags), in fastop()
5084 : "c"(ctxt->src2.val)); in fastop()
5086 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK); in fastop()
5095 ctxt->rip_relative = false; in init_decode_cache()
5096 ctxt->rex_prefix = 0; in init_decode_cache()
5097 ctxt->lock_prefix = 0; in init_decode_cache()
5098 ctxt->rep_prefix = 0; in init_decode_cache()
5099 ctxt->regs_valid = 0; in init_decode_cache()
5100 ctxt->regs_dirty = 0; in init_decode_cache()
5102 ctxt->io_read.pos = 0; in init_decode_cache()
5103 ctxt->io_read.end = 0; in init_decode_cache()
5104 ctxt->mem_read.end = 0; in init_decode_cache()
5109 const struct x86_emulate_ops *ops = ctxt->ops; in x86_emulate_insn()
5111 int saved_dst_type = ctxt->dst.type; in x86_emulate_insn()
5112 bool is_guest_mode = ctxt->ops->is_guest_mode(ctxt); in x86_emulate_insn()
5114 ctxt->mem_read.pos = 0; in x86_emulate_insn()
5117 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) { in x86_emulate_insn()
5122 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) { in x86_emulate_insn()
5127 if (unlikely(ctxt->d & in x86_emulate_insn()
5129 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) || in x86_emulate_insn()
5130 (ctxt->d & Undefined)) { in x86_emulate_insn()
5135 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM))) in x86_emulate_insn()
5136 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) { in x86_emulate_insn()
5141 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) { in x86_emulate_insn()
5146 if (ctxt->d & Mmx) { in x86_emulate_insn()
5154 fetch_possible_mmx_operand(&ctxt->src); in x86_emulate_insn()
5155 fetch_possible_mmx_operand(&ctxt->src2); in x86_emulate_insn()
5156 if (!(ctxt->d & Mov)) in x86_emulate_insn()
5157 fetch_possible_mmx_operand(&ctxt->dst); in x86_emulate_insn()
5160 if (unlikely(is_guest_mode) && ctxt->intercept) { in x86_emulate_insn()
5161 rc = emulator_check_intercept(ctxt, ctxt->intercept, in x86_emulate_insn()
5167 /* Instruction can only be executed in protected mode */ in x86_emulate_insn()
5168 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) { in x86_emulate_insn()
5174 if ((ctxt->d & Priv) && ops->cpl(ctxt)) { in x86_emulate_insn()
5175 if (ctxt->d & PrivUD) in x86_emulate_insn()
5183 if (ctxt->d & CheckPerm) { in x86_emulate_insn()
5184 rc = ctxt->check_perm(ctxt); in x86_emulate_insn()
5189 if (unlikely(is_guest_mode) && (ctxt->d & Intercept)) { in x86_emulate_insn()
5190 rc = emulator_check_intercept(ctxt, ctxt->intercept, in x86_emulate_insn()
5196 if (ctxt->rep_prefix && (ctxt->d & String)) { in x86_emulate_insn()
5200 ctxt->eip = ctxt->_eip; in x86_emulate_insn()
5201 ctxt->eflags &= ~X86_EFLAGS_RF; in x86_emulate_insn()
5207 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) { in x86_emulate_insn()
5208 rc = segmented_read(ctxt, ctxt->src.addr.mem, in x86_emulate_insn()
5209 ctxt->src.valptr, ctxt->src.bytes); in x86_emulate_insn()
5212 ctxt->src.orig_val64 = ctxt->src.val64; in x86_emulate_insn()
5215 if (ctxt->src2.type == OP_MEM) { in x86_emulate_insn()
5216 rc = segmented_read(ctxt, ctxt->src2.addr.mem, in x86_emulate_insn()
5217 &ctxt->src2.val, ctxt->src2.bytes); in x86_emulate_insn()
5222 if ((ctxt->d & DstMask) == ImplicitOps) in x86_emulate_insn()
5226 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) { in x86_emulate_insn()
5227 /* optimisation - avoid slow emulated read if Mov */ in x86_emulate_insn()
5228 rc = segmented_read(ctxt, ctxt->dst.addr.mem, in x86_emulate_insn()
5229 &ctxt->dst.val, ctxt->dst.bytes); in x86_emulate_insn()
5231 if (!(ctxt->d & NoWrite) && in x86_emulate_insn()
5233 ctxt->exception.vector == PF_VECTOR) in x86_emulate_insn()
5234 ctxt->exception.error_code |= PFERR_WRITE_MASK; in x86_emulate_insn()
5238 /* Copy full 64-bit value for CMPXCHG8B. */ in x86_emulate_insn()
5239 ctxt->dst.orig_val64 = ctxt->dst.val64; in x86_emulate_insn()
5243 if (unlikely(is_guest_mode) && (ctxt->d & Intercept)) { in x86_emulate_insn()
5244 rc = emulator_check_intercept(ctxt, ctxt->intercept, in x86_emulate_insn()
5250 if (ctxt->rep_prefix && (ctxt->d & String)) in x86_emulate_insn()
5251 ctxt->eflags |= X86_EFLAGS_RF; in x86_emulate_insn()
5253 ctxt->eflags &= ~X86_EFLAGS_RF; in x86_emulate_insn()
5255 if (ctxt->execute) { in x86_emulate_insn()
5256 if (ctxt->d & Fastop) in x86_emulate_insn()
5257 rc = fastop(ctxt, ctxt->fop); in x86_emulate_insn()
5259 rc = ctxt->execute(ctxt); in x86_emulate_insn()
5265 if (ctxt->opcode_len == 2) in x86_emulate_insn()
5267 else if (ctxt->opcode_len == 3) in x86_emulate_insn()
5270 switch (ctxt->b) { in x86_emulate_insn()
5272 if (test_cc(ctxt->b, ctxt->eflags)) in x86_emulate_insn()
5273 rc = jmp_rel(ctxt, ctxt->src.val); in x86_emulate_insn()
5276 ctxt->dst.val = ctxt->src.addr.mem.ea; in x86_emulate_insn()
5279 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX)) in x86_emulate_insn()
5280 ctxt->dst.type = OP_NONE; in x86_emulate_insn()
5285 switch (ctxt->op_bytes) { in x86_emulate_insn()
5286 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break; in x86_emulate_insn()
5287 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break; in x86_emulate_insn()
5288 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break; in x86_emulate_insn()
5295 rc = emulate_int(ctxt, ctxt->src.val); in x86_emulate_insn()
5298 if (ctxt->eflags & X86_EFLAGS_OF) in x86_emulate_insn()
5303 rc = jmp_rel(ctxt, ctxt->src.val); in x86_emulate_insn()
5304 ctxt->dst.type = OP_NONE; /* Disable writeback. */ in x86_emulate_insn()
5307 ctxt->ops->halt(ctxt); in x86_emulate_insn()
5311 ctxt->eflags ^= X86_EFLAGS_CF; in x86_emulate_insn()
5314 ctxt->eflags &= ~X86_EFLAGS_CF; in x86_emulate_insn()
5317 ctxt->eflags |= X86_EFLAGS_CF; in x86_emulate_insn()
5320 ctxt->eflags &= ~X86_EFLAGS_DF; in x86_emulate_insn()
5323 ctxt->eflags |= X86_EFLAGS_DF; in x86_emulate_insn()
5333 if (ctxt->d & SrcWrite) { in x86_emulate_insn()
5334 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR); in x86_emulate_insn()
5335 rc = writeback(ctxt, &ctxt->src); in x86_emulate_insn()
5339 if (!(ctxt->d & NoWrite)) { in x86_emulate_insn()
5340 rc = writeback(ctxt, &ctxt->dst); in x86_emulate_insn()
5349 ctxt->dst.type = saved_dst_type; in x86_emulate_insn()
5351 if ((ctxt->d & SrcMask) == SrcSI) in x86_emulate_insn()
5352 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src); in x86_emulate_insn()
5354 if ((ctxt->d & DstMask) == DstDI) in x86_emulate_insn()
5355 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst); in x86_emulate_insn()
5357 if (ctxt->rep_prefix && (ctxt->d & String)) { in x86_emulate_insn()
5359 struct read_cache *r = &ctxt->io_read; in x86_emulate_insn()
5360 if ((ctxt->d & SrcMask) == SrcSI) in x86_emulate_insn()
5361 count = ctxt->src.count; in x86_emulate_insn()
5363 count = ctxt->dst.count; in x86_emulate_insn()
5364 register_address_increment(ctxt, VCPU_REGS_RCX, -count); in x86_emulate_insn()
5368 * Re-enter guest when pio read ahead buffer is empty in x86_emulate_insn()
5371 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) && in x86_emulate_insn()
5372 (r->end == 0 || r->end != r->pos)) { in x86_emulate_insn()
5378 ctxt->mem_read.end = 0; in x86_emulate_insn()
5384 ctxt->eflags &= ~X86_EFLAGS_RF; in x86_emulate_insn()
5387 ctxt->eip = ctxt->_eip; in x86_emulate_insn()
5388 if (ctxt->mode != X86EMUL_MODE_PROT64) in x86_emulate_insn()
5389 ctxt->eip = (u32)ctxt->_eip; in x86_emulate_insn()
5393 if (KVM_EMULATOR_BUG_ON(ctxt->exception.vector > 0x1f, ctxt)) in x86_emulate_insn()
5395 ctxt->have_exception = true; in x86_emulate_insn()
5406 switch (ctxt->b) { in x86_emulate_insn()
5408 (ctxt->ops->wbinvd)(ctxt); in x86_emulate_insn()
5416 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg); in x86_emulate_insn()
5419 ctxt->dst.val = ops->get_dr(ctxt, ctxt->modrm_reg); in x86_emulate_insn()
5422 if (test_cc(ctxt->b, ctxt->eflags)) in x86_emulate_insn()
5423 ctxt->dst.val = ctxt->src.val; in x86_emulate_insn()
5424 else if (ctxt->op_bytes != 4) in x86_emulate_insn()
5425 ctxt->dst.type = OP_NONE; /* no writeback */ in x86_emulate_insn()
5428 if (test_cc(ctxt->b, ctxt->eflags)) in x86_emulate_insn()
5429 rc = jmp_rel(ctxt, ctxt->src.val); in x86_emulate_insn()
5432 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags); in x86_emulate_insn()
5435 ctxt->dst.bytes = ctxt->op_bytes; in x86_emulate_insn()
5436 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val in x86_emulate_insn()
5437 : (u16) ctxt->src.val; in x86_emulate_insn()
5440 ctxt->dst.bytes = ctxt->op_bytes; in x86_emulate_insn()
5441 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val : in x86_emulate_insn()
5442 (s16) ctxt->src.val; in x86_emulate_insn()
5471 if (ctxt->rep_prefix && (ctxt->d & String)) in emulator_can_use_gpa()
5474 if (ctxt->d & TwoMemOp) in emulator_can_use_gpa()