1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstImmUByte (OpImmUByte << DstShift)
90 #define DstDX (OpDX << DstShift)
91 #define DstAccLo (OpAccLo << DstShift)
92 #define DstMask (OpMask << DstShift)
93 /* Source operand type. */
95 #define SrcNone (OpNone << SrcShift)
96 #define SrcReg (OpReg << SrcShift)
97 #define SrcMem (OpMem << SrcShift)
98 #define SrcMem16 (OpMem16 << SrcShift)
99 #define SrcMem32 (OpMem32 << SrcShift)
100 #define SrcImm (OpImm << SrcShift)
101 #define SrcImmByte (OpImmByte << SrcShift)
102 #define SrcOne (OpOne << SrcShift)
103 #define SrcImmUByte (OpImmUByte << SrcShift)
104 #define SrcImmU (OpImmU << SrcShift)
105 #define SrcSI (OpSI << SrcShift)
106 #define SrcXLat (OpXLat << SrcShift)
107 #define SrcImmFAddr (OpImmFAddr << SrcShift)
108 #define SrcMemFAddr (OpMemFAddr << SrcShift)
109 #define SrcAcc (OpAcc << SrcShift)
110 #define SrcImmU16 (OpImmU16 << SrcShift)
111 #define SrcImm64 (OpImm64 << SrcShift)
112 #define SrcDX (OpDX << SrcShift)
113 #define SrcMem8 (OpMem8 << SrcShift)
114 #define SrcAccHi (OpAccHi << SrcShift)
115 #define SrcMask (OpMask << SrcShift)
116 #define BitOp (1<<11)
117 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
118 #define String (1<<13) /* String instruction (rep capable) */
119 #define Stack (1<<14) /* Stack instruction (push/pop) */
120 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
121 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
122 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
123 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
124 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
125 #define Escape (5<<15) /* Escape to coprocessor instruction */
126 #define Sse (1<<18) /* SSE Vector instruction */
127 /* Generic ModRM decode. */
128 #define ModRM (1<<19)
129 /* Destination is only written; never read. */
132 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
133 #define VendorSpecific (1<<22) /* Vendor specific instruction */
134 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
135 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
136 #define Undefined (1<<25) /* No Such Instruction */
137 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
138 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
140 #define PageTable (1 << 29) /* instruction used to write page table */
141 #define NotImpl (1 << 30) /* instruction is not implemented */
142 /* Source 2 operand type */
143 #define Src2Shift (31)
144 #define Src2None (OpNone << Src2Shift)
145 #define Src2Mem (OpMem << Src2Shift)
146 #define Src2CL (OpCL << Src2Shift)
147 #define Src2ImmByte (OpImmByte << Src2Shift)
148 #define Src2One (OpOne << Src2Shift)
149 #define Src2Imm (OpImm << Src2Shift)
150 #define Src2ES (OpES << Src2Shift)
151 #define Src2CS (OpCS << Src2Shift)
152 #define Src2SS (OpSS << Src2Shift)
153 #define Src2DS (OpDS << Src2Shift)
154 #define Src2FS (OpFS << Src2Shift)
155 #define Src2GS (OpGS << Src2Shift)
156 #define Src2Mask (OpMask << Src2Shift)
157 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
158 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
159 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
160 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
161 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
162 #define NoWrite ((u64)1 << 45) /* No writeback */
163 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
165 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
167 #define X2(x...) x, x
168 #define X3(x...) X2(x), x
169 #define X4(x...) X2(x), X2(x)
170 #define X5(x...) X4(x), x
171 #define X6(x...) X4(x), X2(x)
172 #define X7(x...) X4(x), X3(x)
173 #define X8(x...) X4(x), X4(x)
174 #define X16(x...) X8(x), X8(x)
176 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
177 #define FASTOP_SIZE 8
180 * fastop functions have a special calling convention:
185 * flags: rflags (in/out)
186 * ex: rsi (in:fastop pointer, out:zero if exception)
188 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
189 * different operand sizes can be reached by calculation, rather than a jump
190 * table (which would be bigger than the code).
192 * fastop functions are declared as taking a never-defined fastop parameter,
193 * so they can't be called from C directly.
202 int (*execute)(struct x86_emulate_ctxt *ctxt);
203 const struct opcode *group;
204 const struct group_dual *gdual;
205 const struct gprefix *gprefix;
206 const struct escape *esc;
207 void (*fastop)(struct fastop *fake);
209 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
213 struct opcode mod012[8];
214 struct opcode mod3[8];
218 struct opcode pfx_no;
219 struct opcode pfx_66;
220 struct opcode pfx_f2;
221 struct opcode pfx_f3;
226 struct opcode high[64];
229 /* EFLAGS bit definitions. */
230 #define EFLG_ID (1<<21)
231 #define EFLG_VIP (1<<20)
232 #define EFLG_VIF (1<<19)
233 #define EFLG_AC (1<<18)
234 #define EFLG_VM (1<<17)
235 #define EFLG_RF (1<<16)
236 #define EFLG_IOPL (3<<12)
237 #define EFLG_NT (1<<14)
238 #define EFLG_OF (1<<11)
239 #define EFLG_DF (1<<10)
240 #define EFLG_IF (1<<9)
241 #define EFLG_TF (1<<8)
242 #define EFLG_SF (1<<7)
243 #define EFLG_ZF (1<<6)
244 #define EFLG_AF (1<<4)
245 #define EFLG_PF (1<<2)
246 #define EFLG_CF (1<<0)
248 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
249 #define EFLG_RESERVED_ONE_MASK 2
251 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
253 if (!(ctxt->regs_valid & (1 << nr))) {
254 ctxt->regs_valid |= 1 << nr;
255 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
257 return ctxt->_regs[nr];
260 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
262 ctxt->regs_valid |= 1 << nr;
263 ctxt->regs_dirty |= 1 << nr;
264 return &ctxt->_regs[nr];
267 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
270 return reg_write(ctxt, nr);
273 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
277 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
278 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
281 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
283 ctxt->regs_dirty = 0;
284 ctxt->regs_valid = 0;
288 * These EFLAGS bits are restored from saved value during emulation, and
289 * any changes are written back to the saved value after emulation.
291 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
299 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
301 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
302 #define FOP_RET "ret \n\t"
304 #define FOP_START(op) \
305 extern void em_##op(struct fastop *fake); \
306 asm(".pushsection .text, \"ax\" \n\t" \
307 ".global em_" #op " \n\t" \
314 #define FOPNOP() FOP_ALIGN FOP_RET
316 #define FOP1E(op, dst) \
317 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
319 #define FOP1EEX(op, dst) \
320 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
322 #define FASTOP1(op) \
327 ON64(FOP1E(op##q, rax)) \
330 /* 1-operand, using src2 (for MUL/DIV r/m) */
331 #define FASTOP1SRC2(op, name) \
336 ON64(FOP1E(op, rcx)) \
339 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
340 #define FASTOP1SRC2EX(op, name) \
345 ON64(FOP1EEX(op, rcx)) \
348 #define FOP2E(op, dst, src) \
349 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
351 #define FASTOP2(op) \
353 FOP2E(op##b, al, dl) \
354 FOP2E(op##w, ax, dx) \
355 FOP2E(op##l, eax, edx) \
356 ON64(FOP2E(op##q, rax, rdx)) \
359 /* 2 operand, word only */
360 #define FASTOP2W(op) \
363 FOP2E(op##w, ax, dx) \
364 FOP2E(op##l, eax, edx) \
365 ON64(FOP2E(op##q, rax, rdx)) \
368 /* 2 operand, src is CL */
369 #define FASTOP2CL(op) \
371 FOP2E(op##b, al, cl) \
372 FOP2E(op##w, ax, cl) \
373 FOP2E(op##l, eax, cl) \
374 ON64(FOP2E(op##q, rax, cl)) \
377 #define FOP3E(op, dst, src, src2) \
378 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
380 /* 3-operand, word-only, src2=cl */
381 #define FASTOP3WCL(op) \
384 FOP3E(op##w, ax, dx, cl) \
385 FOP3E(op##l, eax, edx, cl) \
386 ON64(FOP3E(op##q, rax, rdx, cl)) \
389 /* Special case for SETcc - 1 instruction per cc */
390 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
392 asm(".global kvm_fastop_exception \n"
393 "kvm_fastop_exception: xor %esi, %esi; ret");
414 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
417 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
418 enum x86_intercept intercept,
419 enum x86_intercept_stage stage)
421 struct x86_instruction_info info = {
422 .intercept = intercept,
423 .rep_prefix = ctxt->rep_prefix,
424 .modrm_mod = ctxt->modrm_mod,
425 .modrm_reg = ctxt->modrm_reg,
426 .modrm_rm = ctxt->modrm_rm,
427 .src_val = ctxt->src.val64,
428 .src_bytes = ctxt->src.bytes,
429 .dst_bytes = ctxt->dst.bytes,
430 .ad_bytes = ctxt->ad_bytes,
431 .next_rip = ctxt->eip,
434 return ctxt->ops->intercept(ctxt, &info, stage);
437 static void assign_masked(ulong *dest, ulong src, ulong mask)
439 *dest = (*dest & ~mask) | (src & mask);
442 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
444 return (1UL << (ctxt->ad_bytes << 3)) - 1;
447 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
450 struct desc_struct ss;
452 if (ctxt->mode == X86EMUL_MODE_PROT64)
454 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
455 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
458 static int stack_size(struct x86_emulate_ctxt *ctxt)
460 return (__fls(stack_mask(ctxt)) + 1) >> 3;
463 /* Access/update address held in a register, based on addressing mode. */
464 static inline unsigned long
465 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
467 if (ctxt->ad_bytes == sizeof(unsigned long))
470 return reg & ad_mask(ctxt);
473 static inline unsigned long
474 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
476 return address_mask(ctxt, reg);
479 static void masked_increment(ulong *reg, ulong mask, int inc)
481 assign_masked(reg, *reg + inc, mask);
485 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
489 if (ctxt->ad_bytes == sizeof(unsigned long))
492 mask = ad_mask(ctxt);
493 masked_increment(reg, mask, inc);
496 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
498 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
501 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
503 register_address_increment(ctxt, &ctxt->_eip, rel);
506 static u32 desc_limit_scaled(struct desc_struct *desc)
508 u32 limit = get_desc_limit(desc);
510 return desc->g ? (limit << 12) | 0xfff : limit;
513 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
515 ctxt->has_seg_override = true;
516 ctxt->seg_override = seg;
519 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
521 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
524 return ctxt->ops->get_cached_segment_base(ctxt, seg);
527 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
529 if (!ctxt->has_seg_override)
532 return ctxt->seg_override;
535 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
536 u32 error, bool valid)
538 ctxt->exception.vector = vec;
539 ctxt->exception.error_code = error;
540 ctxt->exception.error_code_valid = valid;
541 return X86EMUL_PROPAGATE_FAULT;
544 static int emulate_db(struct x86_emulate_ctxt *ctxt)
546 return emulate_exception(ctxt, DB_VECTOR, 0, false);
549 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
551 return emulate_exception(ctxt, GP_VECTOR, err, true);
554 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
556 return emulate_exception(ctxt, SS_VECTOR, err, true);
559 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
561 return emulate_exception(ctxt, UD_VECTOR, 0, false);
564 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
566 return emulate_exception(ctxt, TS_VECTOR, err, true);
569 static int emulate_de(struct x86_emulate_ctxt *ctxt)
571 return emulate_exception(ctxt, DE_VECTOR, 0, false);
574 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
576 return emulate_exception(ctxt, NM_VECTOR, 0, false);
579 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
582 struct desc_struct desc;
584 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
588 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
593 struct desc_struct desc;
595 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
596 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
600 * x86 defines three classes of vector instructions: explicitly
601 * aligned, explicitly unaligned, and the rest, which change behaviour
602 * depending on whether they're AVX encoded or not.
604 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
605 * subject to the same check.
607 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
609 if (likely(size < 16))
612 if (ctxt->d & Aligned)
614 else if (ctxt->d & Unaligned)
616 else if (ctxt->d & Avx)
622 static int __linearize(struct x86_emulate_ctxt *ctxt,
623 struct segmented_address addr,
624 unsigned size, bool write, bool fetch,
627 struct desc_struct desc;
634 la = seg_base(ctxt, addr.seg) + addr.ea;
635 switch (ctxt->mode) {
636 case X86EMUL_MODE_PROT64:
637 if (((signed long)la << 16) >> 16 != la)
638 return emulate_gp(ctxt, 0);
641 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
645 /* code segment in protected mode or read-only data segment */
646 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
647 || !(desc.type & 2)) && write)
649 /* unreadable code segment */
650 if (!fetch && (desc.type & 8) && !(desc.type & 2))
652 lim = desc_limit_scaled(&desc);
653 if ((desc.type & 8) || !(desc.type & 4)) {
654 /* expand-up segment */
655 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
658 /* expand-down segment */
659 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
661 lim = desc.d ? 0xffffffff : 0xffff;
662 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
665 cpl = ctxt->ops->cpl(ctxt);
666 if (!(desc.type & 8)) {
670 } else if ((desc.type & 8) && !(desc.type & 4)) {
671 /* nonconforming code segment */
674 } else if ((desc.type & 8) && (desc.type & 4)) {
675 /* conforming code segment */
681 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
683 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
684 return emulate_gp(ctxt, 0);
686 return X86EMUL_CONTINUE;
688 if (addr.seg == VCPU_SREG_SS)
689 return emulate_ss(ctxt, sel);
691 return emulate_gp(ctxt, sel);
694 static int linearize(struct x86_emulate_ctxt *ctxt,
695 struct segmented_address addr,
696 unsigned size, bool write,
699 return __linearize(ctxt, addr, size, write, false, linear);
703 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
704 struct segmented_address addr,
711 rc = linearize(ctxt, addr, size, false, &linear);
712 if (rc != X86EMUL_CONTINUE)
714 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
718 * Fetch the next byte of the instruction being emulated which is pointed to
719 * by ctxt->_eip, then increment ctxt->_eip.
721 * Also prefetch the remaining bytes of the instruction without crossing page
722 * boundary if they are not in fetch_cache yet.
724 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
726 struct fetch_cache *fc = &ctxt->fetch;
730 if (ctxt->_eip == fc->end) {
731 unsigned long linear;
732 struct segmented_address addr = { .seg = VCPU_SREG_CS,
734 cur_size = fc->end - fc->start;
735 size = min(15UL - cur_size,
736 PAGE_SIZE - offset_in_page(ctxt->_eip));
737 rc = __linearize(ctxt, addr, size, false, true, &linear);
738 if (unlikely(rc != X86EMUL_CONTINUE))
740 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
741 size, &ctxt->exception);
742 if (unlikely(rc != X86EMUL_CONTINUE))
746 *dest = fc->data[ctxt->_eip - fc->start];
748 return X86EMUL_CONTINUE;
751 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
752 void *dest, unsigned size)
756 /* x86 instructions are limited to 15 bytes. */
757 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
758 return X86EMUL_UNHANDLEABLE;
760 rc = do_insn_fetch_byte(ctxt, dest++);
761 if (rc != X86EMUL_CONTINUE)
764 return X86EMUL_CONTINUE;
767 /* Fetch next part of the instruction being emulated. */
768 #define insn_fetch(_type, _ctxt) \
769 ({ unsigned long _x; \
770 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
771 if (rc != X86EMUL_CONTINUE) \
776 #define insn_fetch_arr(_arr, _size, _ctxt) \
777 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
778 if (rc != X86EMUL_CONTINUE) \
783 * Given the 'reg' portion of a ModRM byte, and a register block, return a
784 * pointer into the block that addresses the relevant register.
785 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
787 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
792 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
793 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
795 p = reg_rmw(ctxt, modrm_reg);
799 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
800 struct segmented_address addr,
801 u16 *size, unsigned long *address, int op_bytes)
808 rc = segmented_read_std(ctxt, addr, size, 2);
809 if (rc != X86EMUL_CONTINUE)
812 rc = segmented_read_std(ctxt, addr, address, op_bytes);
826 FASTOP1SRC2(mul, mul_ex);
827 FASTOP1SRC2(imul, imul_ex);
828 FASTOP1SRC2EX(div, div_ex);
829 FASTOP1SRC2EX(idiv, idiv_ex);
858 static u8 test_cc(unsigned int condition, unsigned long flags)
861 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
863 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
864 asm("push %[flags]; popf; call *%[fastop]"
865 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
869 static void fetch_register_operand(struct operand *op)
873 op->val = *(u8 *)op->addr.reg;
876 op->val = *(u16 *)op->addr.reg;
879 op->val = *(u32 *)op->addr.reg;
882 op->val = *(u64 *)op->addr.reg;
887 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
889 ctxt->ops->get_fpu(ctxt);
891 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
892 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
893 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
894 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
895 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
896 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
897 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
898 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
900 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
901 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
902 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
903 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
904 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
905 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
906 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
907 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
911 ctxt->ops->put_fpu(ctxt);
914 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
917 ctxt->ops->get_fpu(ctxt);
919 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
920 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
921 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
922 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
923 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
924 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
925 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
926 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
928 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
929 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
930 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
931 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
932 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
933 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
934 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
935 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
939 ctxt->ops->put_fpu(ctxt);
942 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
944 ctxt->ops->get_fpu(ctxt);
946 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
947 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
948 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
949 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
950 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
951 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
952 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
953 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
956 ctxt->ops->put_fpu(ctxt);
959 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
961 ctxt->ops->get_fpu(ctxt);
963 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
964 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
965 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
966 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
967 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
968 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
969 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
970 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
973 ctxt->ops->put_fpu(ctxt);
976 static int em_fninit(struct x86_emulate_ctxt *ctxt)
978 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
979 return emulate_nm(ctxt);
981 ctxt->ops->get_fpu(ctxt);
982 asm volatile("fninit");
983 ctxt->ops->put_fpu(ctxt);
984 return X86EMUL_CONTINUE;
987 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
991 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
992 return emulate_nm(ctxt);
994 ctxt->ops->get_fpu(ctxt);
995 asm volatile("fnstcw %0": "+m"(fcw));
996 ctxt->ops->put_fpu(ctxt);
998 /* force 2 byte destination */
1000 ctxt->dst.val = fcw;
1002 return X86EMUL_CONTINUE;
1005 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1009 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1010 return emulate_nm(ctxt);
1012 ctxt->ops->get_fpu(ctxt);
1013 asm volatile("fnstsw %0": "+m"(fsw));
1014 ctxt->ops->put_fpu(ctxt);
1016 /* force 2 byte destination */
1017 ctxt->dst.bytes = 2;
1018 ctxt->dst.val = fsw;
1020 return X86EMUL_CONTINUE;
1023 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1026 unsigned reg = ctxt->modrm_reg;
1027 int highbyte_regs = ctxt->rex_prefix == 0;
1029 if (!(ctxt->d & ModRM))
1030 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1032 if (ctxt->d & Sse) {
1036 read_sse_reg(ctxt, &op->vec_val, reg);
1039 if (ctxt->d & Mmx) {
1048 if (ctxt->d & ByteOp) {
1049 op->addr.reg = decode_register(ctxt, reg, highbyte_regs);
1052 op->addr.reg = decode_register(ctxt, reg, 0);
1053 op->bytes = ctxt->op_bytes;
1055 fetch_register_operand(op);
1056 op->orig_val = op->val;
1059 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1061 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1062 ctxt->modrm_seg = VCPU_SREG_SS;
1065 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1069 int index_reg = 0, base_reg = 0, scale;
1070 int rc = X86EMUL_CONTINUE;
1073 if (ctxt->rex_prefix) {
1074 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
1075 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
1076 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1079 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1080 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1081 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1082 ctxt->modrm_seg = VCPU_SREG_DS;
1084 if (ctxt->modrm_mod == 3) {
1085 int highbyte_regs = ctxt->rex_prefix == 0;
1088 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1089 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1090 highbyte_regs && (ctxt->d & ByteOp));
1091 if (ctxt->d & Sse) {
1094 op->addr.xmm = ctxt->modrm_rm;
1095 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1098 if (ctxt->d & Mmx) {
1101 op->addr.xmm = ctxt->modrm_rm & 7;
1104 fetch_register_operand(op);
1110 if (ctxt->ad_bytes == 2) {
1111 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1112 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1113 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1114 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1116 /* 16-bit ModR/M decode. */
1117 switch (ctxt->modrm_mod) {
1119 if (ctxt->modrm_rm == 6)
1120 modrm_ea += insn_fetch(u16, ctxt);
1123 modrm_ea += insn_fetch(s8, ctxt);
1126 modrm_ea += insn_fetch(u16, ctxt);
1129 switch (ctxt->modrm_rm) {
1131 modrm_ea += bx + si;
1134 modrm_ea += bx + di;
1137 modrm_ea += bp + si;
1140 modrm_ea += bp + di;
1149 if (ctxt->modrm_mod != 0)
1156 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1157 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1158 ctxt->modrm_seg = VCPU_SREG_SS;
1159 modrm_ea = (u16)modrm_ea;
1161 /* 32/64-bit ModR/M decode. */
1162 if ((ctxt->modrm_rm & 7) == 4) {
1163 sib = insn_fetch(u8, ctxt);
1164 index_reg |= (sib >> 3) & 7;
1165 base_reg |= sib & 7;
1168 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1169 modrm_ea += insn_fetch(s32, ctxt);
1171 modrm_ea += reg_read(ctxt, base_reg);
1172 adjust_modrm_seg(ctxt, base_reg);
1175 modrm_ea += reg_read(ctxt, index_reg) << scale;
1176 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1177 if (ctxt->mode == X86EMUL_MODE_PROT64)
1178 ctxt->rip_relative = 1;
1180 base_reg = ctxt->modrm_rm;
1181 modrm_ea += reg_read(ctxt, base_reg);
1182 adjust_modrm_seg(ctxt, base_reg);
1184 switch (ctxt->modrm_mod) {
1186 if (ctxt->modrm_rm == 5)
1187 modrm_ea += insn_fetch(s32, ctxt);
1190 modrm_ea += insn_fetch(s8, ctxt);
1193 modrm_ea += insn_fetch(s32, ctxt);
1197 op->addr.mem.ea = modrm_ea;
1202 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1205 int rc = X86EMUL_CONTINUE;
1208 switch (ctxt->ad_bytes) {
1210 op->addr.mem.ea = insn_fetch(u16, ctxt);
1213 op->addr.mem.ea = insn_fetch(u32, ctxt);
1216 op->addr.mem.ea = insn_fetch(u64, ctxt);
1223 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1227 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1228 mask = ~(ctxt->dst.bytes * 8 - 1);
1230 if (ctxt->src.bytes == 2)
1231 sv = (s16)ctxt->src.val & (s16)mask;
1232 else if (ctxt->src.bytes == 4)
1233 sv = (s32)ctxt->src.val & (s32)mask;
1235 ctxt->dst.addr.mem.ea += (sv >> 3);
1238 /* only subword offset */
1239 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1242 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1243 unsigned long addr, void *dest, unsigned size)
1246 struct read_cache *mc = &ctxt->mem_read;
1248 if (mc->pos < mc->end)
1251 WARN_ON((mc->end + size) >= sizeof(mc->data));
1253 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1255 if (rc != X86EMUL_CONTINUE)
1261 memcpy(dest, mc->data + mc->pos, size);
1263 return X86EMUL_CONTINUE;
1266 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1267 struct segmented_address addr,
1274 rc = linearize(ctxt, addr, size, false, &linear);
1275 if (rc != X86EMUL_CONTINUE)
1277 return read_emulated(ctxt, linear, data, size);
1280 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1281 struct segmented_address addr,
1288 rc = linearize(ctxt, addr, size, true, &linear);
1289 if (rc != X86EMUL_CONTINUE)
1291 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1295 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1296 struct segmented_address addr,
1297 const void *orig_data, const void *data,
1303 rc = linearize(ctxt, addr, size, true, &linear);
1304 if (rc != X86EMUL_CONTINUE)
1306 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1307 size, &ctxt->exception);
1310 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1311 unsigned int size, unsigned short port,
1314 struct read_cache *rc = &ctxt->io_read;
1316 if (rc->pos == rc->end) { /* refill pio read ahead */
1317 unsigned int in_page, n;
1318 unsigned int count = ctxt->rep_prefix ?
1319 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1320 in_page = (ctxt->eflags & EFLG_DF) ?
1321 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1322 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1323 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1327 rc->pos = rc->end = 0;
1328 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1333 if (ctxt->rep_prefix && !(ctxt->eflags & EFLG_DF)) {
1334 ctxt->dst.data = rc->data + rc->pos;
1335 ctxt->dst.type = OP_MEM_STR;
1336 ctxt->dst.count = (rc->end - rc->pos) / size;
1339 memcpy(dest, rc->data + rc->pos, size);
1345 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1346 u16 index, struct desc_struct *desc)
1351 ctxt->ops->get_idt(ctxt, &dt);
1353 if (dt.size < index * 8 + 7)
1354 return emulate_gp(ctxt, index << 3 | 0x2);
1356 addr = dt.address + index * 8;
1357 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1361 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1362 u16 selector, struct desc_ptr *dt)
1364 const struct x86_emulate_ops *ops = ctxt->ops;
1366 if (selector & 1 << 2) {
1367 struct desc_struct desc;
1370 memset (dt, 0, sizeof *dt);
1371 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1374 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1375 dt->address = get_desc_base(&desc);
1377 ops->get_gdt(ctxt, dt);
1380 /* allowed just for 8 bytes segments */
1381 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1382 u16 selector, struct desc_struct *desc,
1386 u16 index = selector >> 3;
1389 get_descriptor_table_ptr(ctxt, selector, &dt);
1391 if (dt.size < index * 8 + 7)
1392 return emulate_gp(ctxt, selector & 0xfffc);
1394 *desc_addr_p = addr = dt.address + index * 8;
1395 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1399 /* allowed just for 8 bytes segments */
1400 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1401 u16 selector, struct desc_struct *desc)
1404 u16 index = selector >> 3;
1407 get_descriptor_table_ptr(ctxt, selector, &dt);
1409 if (dt.size < index * 8 + 7)
1410 return emulate_gp(ctxt, selector & 0xfffc);
1412 addr = dt.address + index * 8;
1413 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1417 /* Does not support long mode */
1418 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1419 u16 selector, int seg)
1421 struct desc_struct seg_desc, old_desc;
1423 unsigned err_vec = GP_VECTOR;
1425 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1430 memset(&seg_desc, 0, sizeof seg_desc);
1432 if (ctxt->mode == X86EMUL_MODE_REAL) {
1433 /* set real mode segment descriptor (keep limit etc. for
1435 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1436 set_desc_base(&seg_desc, selector << 4);
1438 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1439 /* VM86 needs a clean new segment descriptor */
1440 set_desc_base(&seg_desc, selector << 4);
1441 set_desc_limit(&seg_desc, 0xffff);
1450 cpl = ctxt->ops->cpl(ctxt);
1452 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1453 if ((seg == VCPU_SREG_CS
1454 || (seg == VCPU_SREG_SS
1455 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1456 || seg == VCPU_SREG_TR)
1460 /* TR should be in GDT only */
1461 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1464 if (null_selector) /* for NULL selector skip all following checks */
1467 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1468 if (ret != X86EMUL_CONTINUE)
1471 err_code = selector & 0xfffc;
1472 err_vec = GP_VECTOR;
1474 /* can't load system descriptor into segment selector */
1475 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1479 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1488 * segment is not a writable data segment or segment
1489 * selector's RPL != CPL or segment selector's RPL != CPL
1491 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1495 if (!(seg_desc.type & 8))
1498 if (seg_desc.type & 4) {
1504 if (rpl > cpl || dpl != cpl)
1507 /* CS(RPL) <- CPL */
1508 selector = (selector & 0xfffc) | cpl;
1511 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1513 old_desc = seg_desc;
1514 seg_desc.type |= 2; /* busy */
1515 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1516 sizeof(seg_desc), &ctxt->exception);
1517 if (ret != X86EMUL_CONTINUE)
1520 case VCPU_SREG_LDTR:
1521 if (seg_desc.s || seg_desc.type != 2)
1524 default: /* DS, ES, FS, or GS */
1526 * segment is not a data or readable code segment or
1527 * ((segment is a data or nonconforming code segment)
1528 * and (both RPL and CPL > DPL))
1530 if ((seg_desc.type & 0xa) == 0x8 ||
1531 (((seg_desc.type & 0xc) != 0xc) &&
1532 (rpl > dpl && cpl > dpl)))
1538 /* mark segment as accessed */
1540 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1541 if (ret != X86EMUL_CONTINUE)
1545 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1546 return X86EMUL_CONTINUE;
1548 emulate_exception(ctxt, err_vec, err_code, true);
1549 return X86EMUL_PROPAGATE_FAULT;
1552 static void write_register_operand(struct operand *op)
1554 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1555 switch (op->bytes) {
1557 *(u8 *)op->addr.reg = (u8)op->val;
1560 *(u16 *)op->addr.reg = (u16)op->val;
1563 *op->addr.reg = (u32)op->val;
1564 break; /* 64b: zero-extend */
1566 *op->addr.reg = op->val;
1571 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1577 write_register_operand(op);
1580 if (ctxt->lock_prefix)
1581 rc = segmented_cmpxchg(ctxt,
1587 rc = segmented_write(ctxt,
1591 if (rc != X86EMUL_CONTINUE)
1595 rc = segmented_write(ctxt,
1598 op->bytes * op->count);
1599 if (rc != X86EMUL_CONTINUE)
1603 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1606 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1614 return X86EMUL_CONTINUE;
1617 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1619 struct segmented_address addr;
1621 rsp_increment(ctxt, -bytes);
1622 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1623 addr.seg = VCPU_SREG_SS;
1625 return segmented_write(ctxt, addr, data, bytes);
1628 static int em_push(struct x86_emulate_ctxt *ctxt)
1630 /* Disable writeback. */
1631 ctxt->dst.type = OP_NONE;
1632 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1635 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1636 void *dest, int len)
1639 struct segmented_address addr;
1641 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1642 addr.seg = VCPU_SREG_SS;
1643 rc = segmented_read(ctxt, addr, dest, len);
1644 if (rc != X86EMUL_CONTINUE)
1647 rsp_increment(ctxt, len);
1651 static int em_pop(struct x86_emulate_ctxt *ctxt)
1653 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1656 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1657 void *dest, int len)
1660 unsigned long val, change_mask;
1661 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1662 int cpl = ctxt->ops->cpl(ctxt);
1664 rc = emulate_pop(ctxt, &val, len);
1665 if (rc != X86EMUL_CONTINUE)
1668 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1669 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1671 switch(ctxt->mode) {
1672 case X86EMUL_MODE_PROT64:
1673 case X86EMUL_MODE_PROT32:
1674 case X86EMUL_MODE_PROT16:
1676 change_mask |= EFLG_IOPL;
1678 change_mask |= EFLG_IF;
1680 case X86EMUL_MODE_VM86:
1682 return emulate_gp(ctxt, 0);
1683 change_mask |= EFLG_IF;
1685 default: /* real mode */
1686 change_mask |= (EFLG_IOPL | EFLG_IF);
1690 *(unsigned long *)dest =
1691 (ctxt->eflags & ~change_mask) | (val & change_mask);
1696 static int em_popf(struct x86_emulate_ctxt *ctxt)
1698 ctxt->dst.type = OP_REG;
1699 ctxt->dst.addr.reg = &ctxt->eflags;
1700 ctxt->dst.bytes = ctxt->op_bytes;
1701 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1704 static int em_enter(struct x86_emulate_ctxt *ctxt)
1707 unsigned frame_size = ctxt->src.val;
1708 unsigned nesting_level = ctxt->src2.val & 31;
1712 return X86EMUL_UNHANDLEABLE;
1714 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1715 rc = push(ctxt, &rbp, stack_size(ctxt));
1716 if (rc != X86EMUL_CONTINUE)
1718 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1720 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1721 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1723 return X86EMUL_CONTINUE;
1726 static int em_leave(struct x86_emulate_ctxt *ctxt)
1728 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1730 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1733 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1735 int seg = ctxt->src2.val;
1737 ctxt->src.val = get_segment_selector(ctxt, seg);
1739 return em_push(ctxt);
1742 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1744 int seg = ctxt->src2.val;
1745 unsigned long selector;
1748 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1749 if (rc != X86EMUL_CONTINUE)
1752 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1756 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1758 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1759 int rc = X86EMUL_CONTINUE;
1760 int reg = VCPU_REGS_RAX;
1762 while (reg <= VCPU_REGS_RDI) {
1763 (reg == VCPU_REGS_RSP) ?
1764 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1767 if (rc != X86EMUL_CONTINUE)
1776 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1778 ctxt->src.val = (unsigned long)ctxt->eflags;
1779 return em_push(ctxt);
1782 static int em_popa(struct x86_emulate_ctxt *ctxt)
1784 int rc = X86EMUL_CONTINUE;
1785 int reg = VCPU_REGS_RDI;
1787 while (reg >= VCPU_REGS_RAX) {
1788 if (reg == VCPU_REGS_RSP) {
1789 rsp_increment(ctxt, ctxt->op_bytes);
1793 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1794 if (rc != X86EMUL_CONTINUE)
1801 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1803 const struct x86_emulate_ops *ops = ctxt->ops;
1810 /* TODO: Add limit checks */
1811 ctxt->src.val = ctxt->eflags;
1813 if (rc != X86EMUL_CONTINUE)
1816 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1818 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1820 if (rc != X86EMUL_CONTINUE)
1823 ctxt->src.val = ctxt->_eip;
1825 if (rc != X86EMUL_CONTINUE)
1828 ops->get_idt(ctxt, &dt);
1830 eip_addr = dt.address + (irq << 2);
1831 cs_addr = dt.address + (irq << 2) + 2;
1833 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1834 if (rc != X86EMUL_CONTINUE)
1837 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1838 if (rc != X86EMUL_CONTINUE)
1841 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1842 if (rc != X86EMUL_CONTINUE)
1850 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1854 invalidate_registers(ctxt);
1855 rc = __emulate_int_real(ctxt, irq);
1856 if (rc == X86EMUL_CONTINUE)
1857 writeback_registers(ctxt);
1861 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1863 switch(ctxt->mode) {
1864 case X86EMUL_MODE_REAL:
1865 return __emulate_int_real(ctxt, irq);
1866 case X86EMUL_MODE_VM86:
1867 case X86EMUL_MODE_PROT16:
1868 case X86EMUL_MODE_PROT32:
1869 case X86EMUL_MODE_PROT64:
1871 /* Protected mode interrupts unimplemented yet */
1872 return X86EMUL_UNHANDLEABLE;
1876 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1878 int rc = X86EMUL_CONTINUE;
1879 unsigned long temp_eip = 0;
1880 unsigned long temp_eflags = 0;
1881 unsigned long cs = 0;
1882 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1883 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1884 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1885 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1887 /* TODO: Add stack limit check */
1889 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1891 if (rc != X86EMUL_CONTINUE)
1894 if (temp_eip & ~0xffff)
1895 return emulate_gp(ctxt, 0);
1897 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1899 if (rc != X86EMUL_CONTINUE)
1902 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1904 if (rc != X86EMUL_CONTINUE)
1907 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1909 if (rc != X86EMUL_CONTINUE)
1912 ctxt->_eip = temp_eip;
1915 if (ctxt->op_bytes == 4)
1916 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1917 else if (ctxt->op_bytes == 2) {
1918 ctxt->eflags &= ~0xffff;
1919 ctxt->eflags |= temp_eflags;
1922 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1923 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1928 static int em_iret(struct x86_emulate_ctxt *ctxt)
1930 switch(ctxt->mode) {
1931 case X86EMUL_MODE_REAL:
1932 return emulate_iret_real(ctxt);
1933 case X86EMUL_MODE_VM86:
1934 case X86EMUL_MODE_PROT16:
1935 case X86EMUL_MODE_PROT32:
1936 case X86EMUL_MODE_PROT64:
1938 /* iret from protected mode unimplemented yet */
1939 return X86EMUL_UNHANDLEABLE;
1943 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1948 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1950 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1951 if (rc != X86EMUL_CONTINUE)
1955 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1956 return X86EMUL_CONTINUE;
1959 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1961 int rc = X86EMUL_CONTINUE;
1963 switch (ctxt->modrm_reg) {
1964 case 2: /* call near abs */ {
1966 old_eip = ctxt->_eip;
1967 ctxt->_eip = ctxt->src.val;
1968 ctxt->src.val = old_eip;
1972 case 4: /* jmp abs */
1973 ctxt->_eip = ctxt->src.val;
1975 case 5: /* jmp far */
1976 rc = em_jmp_far(ctxt);
1985 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
1987 u64 old = ctxt->dst.orig_val64;
1989 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
1990 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
1991 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
1992 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
1993 ctxt->eflags &= ~EFLG_ZF;
1995 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
1996 (u32) reg_read(ctxt, VCPU_REGS_RBX);
1998 ctxt->eflags |= EFLG_ZF;
2000 return X86EMUL_CONTINUE;
2003 static int em_ret(struct x86_emulate_ctxt *ctxt)
2005 ctxt->dst.type = OP_REG;
2006 ctxt->dst.addr.reg = &ctxt->_eip;
2007 ctxt->dst.bytes = ctxt->op_bytes;
2008 return em_pop(ctxt);
2011 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2016 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2017 if (rc != X86EMUL_CONTINUE)
2019 if (ctxt->op_bytes == 4)
2020 ctxt->_eip = (u32)ctxt->_eip;
2021 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2022 if (rc != X86EMUL_CONTINUE)
2024 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2028 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2030 /* Save real source value, then compare EAX against destination. */
2031 ctxt->src.orig_val = ctxt->src.val;
2032 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
2033 fastop(ctxt, em_cmp);
2035 if (ctxt->eflags & EFLG_ZF) {
2036 /* Success: write back to memory. */
2037 ctxt->dst.val = ctxt->src.orig_val;
2039 /* Failure: write the value we saw to EAX. */
2040 ctxt->dst.type = OP_REG;
2041 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2043 return X86EMUL_CONTINUE;
2046 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2048 int seg = ctxt->src2.val;
2052 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2054 rc = load_segment_descriptor(ctxt, sel, seg);
2055 if (rc != X86EMUL_CONTINUE)
2058 ctxt->dst.val = ctxt->src.val;
2063 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2064 struct desc_struct *cs, struct desc_struct *ss)
2066 cs->l = 0; /* will be adjusted later */
2067 set_desc_base(cs, 0); /* flat segment */
2068 cs->g = 1; /* 4kb granularity */
2069 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2070 cs->type = 0x0b; /* Read, Execute, Accessed */
2072 cs->dpl = 0; /* will be adjusted later */
2077 set_desc_base(ss, 0); /* flat segment */
2078 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2079 ss->g = 1; /* 4kb granularity */
2081 ss->type = 0x03; /* Read/Write, Accessed */
2082 ss->d = 1; /* 32bit stack segment */
2089 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2091 u32 eax, ebx, ecx, edx;
2094 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2095 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2096 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2097 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2100 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2102 const struct x86_emulate_ops *ops = ctxt->ops;
2103 u32 eax, ebx, ecx, edx;
2106 * syscall should always be enabled in longmode - so only become
2107 * vendor specific (cpuid) if other modes are active...
2109 if (ctxt->mode == X86EMUL_MODE_PROT64)
2114 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2116 * Intel ("GenuineIntel")
2117 * remark: Intel CPUs only support "syscall" in 64bit
2118 * longmode. Also an 64bit guest with a
2119 * 32bit compat-app running will #UD !! While this
2120 * behaviour can be fixed (by emulating) into AMD
2121 * response - CPUs of AMD can't behave like Intel.
2123 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2124 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2125 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2128 /* AMD ("AuthenticAMD") */
2129 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2130 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2131 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2134 /* AMD ("AMDisbetter!") */
2135 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2136 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2137 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2140 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2144 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2146 const struct x86_emulate_ops *ops = ctxt->ops;
2147 struct desc_struct cs, ss;
2152 /* syscall is not available in real mode */
2153 if (ctxt->mode == X86EMUL_MODE_REAL ||
2154 ctxt->mode == X86EMUL_MODE_VM86)
2155 return emulate_ud(ctxt);
2157 if (!(em_syscall_is_enabled(ctxt)))
2158 return emulate_ud(ctxt);
2160 ops->get_msr(ctxt, MSR_EFER, &efer);
2161 setup_syscalls_segments(ctxt, &cs, &ss);
2163 if (!(efer & EFER_SCE))
2164 return emulate_ud(ctxt);
2166 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2168 cs_sel = (u16)(msr_data & 0xfffc);
2169 ss_sel = (u16)(msr_data + 8);
2171 if (efer & EFER_LMA) {
2175 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2176 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2178 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2179 if (efer & EFER_LMA) {
2180 #ifdef CONFIG_X86_64
2181 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2184 ctxt->mode == X86EMUL_MODE_PROT64 ?
2185 MSR_LSTAR : MSR_CSTAR, &msr_data);
2186 ctxt->_eip = msr_data;
2188 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2189 ctxt->eflags &= ~(msr_data | EFLG_RF);
2193 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2194 ctxt->_eip = (u32)msr_data;
2196 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2199 return X86EMUL_CONTINUE;
2202 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2204 const struct x86_emulate_ops *ops = ctxt->ops;
2205 struct desc_struct cs, ss;
2210 ops->get_msr(ctxt, MSR_EFER, &efer);
2211 /* inject #GP if in real mode */
2212 if (ctxt->mode == X86EMUL_MODE_REAL)
2213 return emulate_gp(ctxt, 0);
2216 * Not recognized on AMD in compat mode (but is recognized in legacy
2219 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2220 && !vendor_intel(ctxt))
2221 return emulate_ud(ctxt);
2223 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2224 * Therefore, we inject an #UD.
2226 if (ctxt->mode == X86EMUL_MODE_PROT64)
2227 return emulate_ud(ctxt);
2229 setup_syscalls_segments(ctxt, &cs, &ss);
2231 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2232 switch (ctxt->mode) {
2233 case X86EMUL_MODE_PROT32:
2234 if ((msr_data & 0xfffc) == 0x0)
2235 return emulate_gp(ctxt, 0);
2237 case X86EMUL_MODE_PROT64:
2238 if (msr_data == 0x0)
2239 return emulate_gp(ctxt, 0);
2245 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2246 cs_sel = (u16)msr_data;
2247 cs_sel &= ~SELECTOR_RPL_MASK;
2248 ss_sel = cs_sel + 8;
2249 ss_sel &= ~SELECTOR_RPL_MASK;
2250 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2255 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2256 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2258 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2259 ctxt->_eip = msr_data;
2261 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2262 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2264 return X86EMUL_CONTINUE;
2267 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2269 const struct x86_emulate_ops *ops = ctxt->ops;
2270 struct desc_struct cs, ss;
2273 u16 cs_sel = 0, ss_sel = 0;
2275 /* inject #GP if in real mode or Virtual 8086 mode */
2276 if (ctxt->mode == X86EMUL_MODE_REAL ||
2277 ctxt->mode == X86EMUL_MODE_VM86)
2278 return emulate_gp(ctxt, 0);
2280 setup_syscalls_segments(ctxt, &cs, &ss);
2282 if ((ctxt->rex_prefix & 0x8) != 0x0)
2283 usermode = X86EMUL_MODE_PROT64;
2285 usermode = X86EMUL_MODE_PROT32;
2289 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2291 case X86EMUL_MODE_PROT32:
2292 cs_sel = (u16)(msr_data + 16);
2293 if ((msr_data & 0xfffc) == 0x0)
2294 return emulate_gp(ctxt, 0);
2295 ss_sel = (u16)(msr_data + 24);
2297 case X86EMUL_MODE_PROT64:
2298 cs_sel = (u16)(msr_data + 32);
2299 if (msr_data == 0x0)
2300 return emulate_gp(ctxt, 0);
2301 ss_sel = cs_sel + 8;
2306 cs_sel |= SELECTOR_RPL_MASK;
2307 ss_sel |= SELECTOR_RPL_MASK;
2309 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2310 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2312 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2313 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2315 return X86EMUL_CONTINUE;
2318 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2321 if (ctxt->mode == X86EMUL_MODE_REAL)
2323 if (ctxt->mode == X86EMUL_MODE_VM86)
2325 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2326 return ctxt->ops->cpl(ctxt) > iopl;
2329 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2332 const struct x86_emulate_ops *ops = ctxt->ops;
2333 struct desc_struct tr_seg;
2336 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2337 unsigned mask = (1 << len) - 1;
2340 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2343 if (desc_limit_scaled(&tr_seg) < 103)
2345 base = get_desc_base(&tr_seg);
2346 #ifdef CONFIG_X86_64
2347 base |= ((u64)base3) << 32;
2349 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2350 if (r != X86EMUL_CONTINUE)
2352 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2354 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2355 if (r != X86EMUL_CONTINUE)
2357 if ((perm >> bit_idx) & mask)
2362 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2368 if (emulator_bad_iopl(ctxt))
2369 if (!emulator_io_port_access_allowed(ctxt, port, len))
2372 ctxt->perm_ok = true;
2377 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2378 struct tss_segment_16 *tss)
2380 tss->ip = ctxt->_eip;
2381 tss->flag = ctxt->eflags;
2382 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2383 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2384 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2385 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2386 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2387 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2388 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2389 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2391 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2392 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2393 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2394 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2395 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2398 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2399 struct tss_segment_16 *tss)
2403 ctxt->_eip = tss->ip;
2404 ctxt->eflags = tss->flag | 2;
2405 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2406 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2407 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2408 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2409 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2410 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2411 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2412 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2415 * SDM says that segment selectors are loaded before segment
2418 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2419 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2420 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2421 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2422 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2425 * Now load segment descriptors. If fault happens at this stage
2426 * it is handled in a context of new task
2428 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2429 if (ret != X86EMUL_CONTINUE)
2431 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2432 if (ret != X86EMUL_CONTINUE)
2434 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2435 if (ret != X86EMUL_CONTINUE)
2437 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2438 if (ret != X86EMUL_CONTINUE)
2440 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2441 if (ret != X86EMUL_CONTINUE)
2444 return X86EMUL_CONTINUE;
2447 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2448 u16 tss_selector, u16 old_tss_sel,
2449 ulong old_tss_base, struct desc_struct *new_desc)
2451 const struct x86_emulate_ops *ops = ctxt->ops;
2452 struct tss_segment_16 tss_seg;
2454 u32 new_tss_base = get_desc_base(new_desc);
2456 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2458 if (ret != X86EMUL_CONTINUE)
2459 /* FIXME: need to provide precise fault address */
2462 save_state_to_tss16(ctxt, &tss_seg);
2464 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2466 if (ret != X86EMUL_CONTINUE)
2467 /* FIXME: need to provide precise fault address */
2470 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2472 if (ret != X86EMUL_CONTINUE)
2473 /* FIXME: need to provide precise fault address */
2476 if (old_tss_sel != 0xffff) {
2477 tss_seg.prev_task_link = old_tss_sel;
2479 ret = ops->write_std(ctxt, new_tss_base,
2480 &tss_seg.prev_task_link,
2481 sizeof tss_seg.prev_task_link,
2483 if (ret != X86EMUL_CONTINUE)
2484 /* FIXME: need to provide precise fault address */
2488 return load_state_from_tss16(ctxt, &tss_seg);
2491 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2492 struct tss_segment_32 *tss)
2494 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2495 tss->eip = ctxt->_eip;
2496 tss->eflags = ctxt->eflags;
2497 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2498 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2499 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2500 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2501 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2502 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2503 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2504 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2506 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2507 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2508 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2509 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2510 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2511 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2512 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2515 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2516 struct tss_segment_32 *tss)
2520 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2521 return emulate_gp(ctxt, 0);
2522 ctxt->_eip = tss->eip;
2523 ctxt->eflags = tss->eflags | 2;
2525 /* General purpose registers */
2526 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2527 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2528 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2529 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2530 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2531 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2532 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2533 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2536 * SDM says that segment selectors are loaded before segment
2539 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2540 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2541 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2542 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2543 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2544 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2545 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2548 * If we're switching between Protected Mode and VM86, we need to make
2549 * sure to update the mode before loading the segment descriptors so
2550 * that the selectors are interpreted correctly.
2552 * Need to get rflags to the vcpu struct immediately because it
2553 * influences the CPL which is checked at least when loading the segment
2554 * descriptors and when pushing an error code to the new kernel stack.
2556 * TODO Introduce a separate ctxt->ops->set_cpl callback
2558 if (ctxt->eflags & X86_EFLAGS_VM)
2559 ctxt->mode = X86EMUL_MODE_VM86;
2561 ctxt->mode = X86EMUL_MODE_PROT32;
2563 ctxt->ops->set_rflags(ctxt, ctxt->eflags);
2566 * Now load segment descriptors. If fault happenes at this stage
2567 * it is handled in a context of new task
2569 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2570 if (ret != X86EMUL_CONTINUE)
2572 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2573 if (ret != X86EMUL_CONTINUE)
2575 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2576 if (ret != X86EMUL_CONTINUE)
2578 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2579 if (ret != X86EMUL_CONTINUE)
2581 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2582 if (ret != X86EMUL_CONTINUE)
2584 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2585 if (ret != X86EMUL_CONTINUE)
2587 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2588 if (ret != X86EMUL_CONTINUE)
2591 return X86EMUL_CONTINUE;
2594 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2595 u16 tss_selector, u16 old_tss_sel,
2596 ulong old_tss_base, struct desc_struct *new_desc)
2598 const struct x86_emulate_ops *ops = ctxt->ops;
2599 struct tss_segment_32 tss_seg;
2601 u32 new_tss_base = get_desc_base(new_desc);
2603 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2605 if (ret != X86EMUL_CONTINUE)
2606 /* FIXME: need to provide precise fault address */
2609 save_state_to_tss32(ctxt, &tss_seg);
2611 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2613 if (ret != X86EMUL_CONTINUE)
2614 /* FIXME: need to provide precise fault address */
2617 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2619 if (ret != X86EMUL_CONTINUE)
2620 /* FIXME: need to provide precise fault address */
2623 if (old_tss_sel != 0xffff) {
2624 tss_seg.prev_task_link = old_tss_sel;
2626 ret = ops->write_std(ctxt, new_tss_base,
2627 &tss_seg.prev_task_link,
2628 sizeof tss_seg.prev_task_link,
2630 if (ret != X86EMUL_CONTINUE)
2631 /* FIXME: need to provide precise fault address */
2635 return load_state_from_tss32(ctxt, &tss_seg);
2638 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2639 u16 tss_selector, int idt_index, int reason,
2640 bool has_error_code, u32 error_code)
2642 const struct x86_emulate_ops *ops = ctxt->ops;
2643 struct desc_struct curr_tss_desc, next_tss_desc;
2645 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2646 ulong old_tss_base =
2647 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2651 /* FIXME: old_tss_base == ~0 ? */
2653 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2654 if (ret != X86EMUL_CONTINUE)
2656 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2657 if (ret != X86EMUL_CONTINUE)
2660 /* FIXME: check that next_tss_desc is tss */
2663 * Check privileges. The three cases are task switch caused by...
2665 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2666 * 2. Exception/IRQ/iret: No check is performed
2667 * 3. jmp/call to TSS: Check against DPL of the TSS
2669 if (reason == TASK_SWITCH_GATE) {
2670 if (idt_index != -1) {
2671 /* Software interrupts */
2672 struct desc_struct task_gate_desc;
2675 ret = read_interrupt_descriptor(ctxt, idt_index,
2677 if (ret != X86EMUL_CONTINUE)
2680 dpl = task_gate_desc.dpl;
2681 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2682 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2684 } else if (reason != TASK_SWITCH_IRET) {
2685 int dpl = next_tss_desc.dpl;
2686 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2687 return emulate_gp(ctxt, tss_selector);
2691 desc_limit = desc_limit_scaled(&next_tss_desc);
2692 if (!next_tss_desc.p ||
2693 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2694 desc_limit < 0x2b)) {
2695 emulate_ts(ctxt, tss_selector & 0xfffc);
2696 return X86EMUL_PROPAGATE_FAULT;
2699 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2700 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2701 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2704 if (reason == TASK_SWITCH_IRET)
2705 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2707 /* set back link to prev task only if NT bit is set in eflags
2708 note that old_tss_sel is not used after this point */
2709 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2710 old_tss_sel = 0xffff;
2712 if (next_tss_desc.type & 8)
2713 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2714 old_tss_base, &next_tss_desc);
2716 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2717 old_tss_base, &next_tss_desc);
2718 if (ret != X86EMUL_CONTINUE)
2721 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2722 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2724 if (reason != TASK_SWITCH_IRET) {
2725 next_tss_desc.type |= (1 << 1); /* set busy flag */
2726 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2729 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2730 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2732 if (has_error_code) {
2733 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2734 ctxt->lock_prefix = 0;
2735 ctxt->src.val = (unsigned long) error_code;
2736 ret = em_push(ctxt);
2742 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2743 u16 tss_selector, int idt_index, int reason,
2744 bool has_error_code, u32 error_code)
2748 invalidate_registers(ctxt);
2749 ctxt->_eip = ctxt->eip;
2750 ctxt->dst.type = OP_NONE;
2752 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2753 has_error_code, error_code);
2755 if (rc == X86EMUL_CONTINUE) {
2756 ctxt->eip = ctxt->_eip;
2757 writeback_registers(ctxt);
2760 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2763 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2766 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2768 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2769 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2772 static int em_das(struct x86_emulate_ctxt *ctxt)
2775 bool af, cf, old_cf;
2777 cf = ctxt->eflags & X86_EFLAGS_CF;
2783 af = ctxt->eflags & X86_EFLAGS_AF;
2784 if ((al & 0x0f) > 9 || af) {
2786 cf = old_cf | (al >= 250);
2791 if (old_al > 0x99 || old_cf) {
2797 /* Set PF, ZF, SF */
2798 ctxt->src.type = OP_IMM;
2800 ctxt->src.bytes = 1;
2801 fastop(ctxt, em_or);
2802 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2804 ctxt->eflags |= X86_EFLAGS_CF;
2806 ctxt->eflags |= X86_EFLAGS_AF;
2807 return X86EMUL_CONTINUE;
2810 static int em_aam(struct x86_emulate_ctxt *ctxt)
2814 if (ctxt->src.val == 0)
2815 return emulate_de(ctxt);
2817 al = ctxt->dst.val & 0xff;
2818 ah = al / ctxt->src.val;
2819 al %= ctxt->src.val;
2821 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
2823 /* Set PF, ZF, SF */
2824 ctxt->src.type = OP_IMM;
2826 ctxt->src.bytes = 1;
2827 fastop(ctxt, em_or);
2829 return X86EMUL_CONTINUE;
2832 static int em_aad(struct x86_emulate_ctxt *ctxt)
2834 u8 al = ctxt->dst.val & 0xff;
2835 u8 ah = (ctxt->dst.val >> 8) & 0xff;
2837 al = (al + (ah * ctxt->src.val)) & 0xff;
2839 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
2841 /* Set PF, ZF, SF */
2842 ctxt->src.type = OP_IMM;
2844 ctxt->src.bytes = 1;
2845 fastop(ctxt, em_or);
2847 return X86EMUL_CONTINUE;
2850 static int em_call(struct x86_emulate_ctxt *ctxt)
2852 long rel = ctxt->src.val;
2854 ctxt->src.val = (unsigned long)ctxt->_eip;
2856 return em_push(ctxt);
2859 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2865 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2866 old_eip = ctxt->_eip;
2868 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2869 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2870 return X86EMUL_CONTINUE;
2873 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2875 ctxt->src.val = old_cs;
2877 if (rc != X86EMUL_CONTINUE)
2880 ctxt->src.val = old_eip;
2881 return em_push(ctxt);
2884 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2888 ctxt->dst.type = OP_REG;
2889 ctxt->dst.addr.reg = &ctxt->_eip;
2890 ctxt->dst.bytes = ctxt->op_bytes;
2891 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2892 if (rc != X86EMUL_CONTINUE)
2894 rsp_increment(ctxt, ctxt->src.val);
2895 return X86EMUL_CONTINUE;
2898 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2900 /* Write back the register source. */
2901 ctxt->src.val = ctxt->dst.val;
2902 write_register_operand(&ctxt->src);
2904 /* Write back the memory destination with implicit LOCK prefix. */
2905 ctxt->dst.val = ctxt->src.orig_val;
2906 ctxt->lock_prefix = 1;
2907 return X86EMUL_CONTINUE;
2910 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2912 ctxt->dst.val = ctxt->src2.val;
2913 return fastop(ctxt, em_imul);
2916 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2918 ctxt->dst.type = OP_REG;
2919 ctxt->dst.bytes = ctxt->src.bytes;
2920 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
2921 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2923 return X86EMUL_CONTINUE;
2926 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2930 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2931 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
2932 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
2933 return X86EMUL_CONTINUE;
2936 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
2940 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
2941 return emulate_gp(ctxt, 0);
2942 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
2943 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
2944 return X86EMUL_CONTINUE;
2947 static int em_mov(struct x86_emulate_ctxt *ctxt)
2949 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
2950 return X86EMUL_CONTINUE;
2953 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
2955 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
2956 return emulate_gp(ctxt, 0);
2958 /* Disable writeback. */
2959 ctxt->dst.type = OP_NONE;
2960 return X86EMUL_CONTINUE;
2963 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
2967 if (ctxt->mode == X86EMUL_MODE_PROT64)
2968 val = ctxt->src.val & ~0ULL;
2970 val = ctxt->src.val & ~0U;
2972 /* #UD condition is already handled. */
2973 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
2974 return emulate_gp(ctxt, 0);
2976 /* Disable writeback. */
2977 ctxt->dst.type = OP_NONE;
2978 return X86EMUL_CONTINUE;
2981 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
2985 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
2986 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
2987 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
2988 return emulate_gp(ctxt, 0);
2990 return X86EMUL_CONTINUE;
2993 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
2997 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
2998 return emulate_gp(ctxt, 0);
3000 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3001 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3002 return X86EMUL_CONTINUE;
3005 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3007 if (ctxt->modrm_reg > VCPU_SREG_GS)
3008 return emulate_ud(ctxt);
3010 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3011 return X86EMUL_CONTINUE;
3014 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3016 u16 sel = ctxt->src.val;
3018 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3019 return emulate_ud(ctxt);
3021 if (ctxt->modrm_reg == VCPU_SREG_SS)
3022 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3024 /* Disable writeback. */
3025 ctxt->dst.type = OP_NONE;
3026 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3029 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3031 u16 sel = ctxt->src.val;
3033 /* Disable writeback. */
3034 ctxt->dst.type = OP_NONE;
3035 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3038 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3040 u16 sel = ctxt->src.val;
3042 /* Disable writeback. */
3043 ctxt->dst.type = OP_NONE;
3044 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3047 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3052 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3053 if (rc == X86EMUL_CONTINUE)
3054 ctxt->ops->invlpg(ctxt, linear);
3055 /* Disable writeback. */
3056 ctxt->dst.type = OP_NONE;
3057 return X86EMUL_CONTINUE;
3060 static int em_clts(struct x86_emulate_ctxt *ctxt)
3064 cr0 = ctxt->ops->get_cr(ctxt, 0);
3066 ctxt->ops->set_cr(ctxt, 0, cr0);
3067 return X86EMUL_CONTINUE;
3070 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3074 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3075 return X86EMUL_UNHANDLEABLE;
3077 rc = ctxt->ops->fix_hypercall(ctxt);
3078 if (rc != X86EMUL_CONTINUE)
3081 /* Let the processor re-execute the fixed hypercall */
3082 ctxt->_eip = ctxt->eip;
3083 /* Disable writeback. */
3084 ctxt->dst.type = OP_NONE;
3085 return X86EMUL_CONTINUE;
3088 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3089 void (*get)(struct x86_emulate_ctxt *ctxt,
3090 struct desc_ptr *ptr))
3092 struct desc_ptr desc_ptr;
3094 if (ctxt->mode == X86EMUL_MODE_PROT64)
3096 get(ctxt, &desc_ptr);
3097 if (ctxt->op_bytes == 2) {
3099 desc_ptr.address &= 0x00ffffff;
3101 /* Disable writeback. */
3102 ctxt->dst.type = OP_NONE;
3103 return segmented_write(ctxt, ctxt->dst.addr.mem,
3104 &desc_ptr, 2 + ctxt->op_bytes);
3107 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3109 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3112 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3114 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3117 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3119 struct desc_ptr desc_ptr;
3122 if (ctxt->mode == X86EMUL_MODE_PROT64)
3124 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3125 &desc_ptr.size, &desc_ptr.address,
3127 if (rc != X86EMUL_CONTINUE)
3129 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3130 /* Disable writeback. */
3131 ctxt->dst.type = OP_NONE;
3132 return X86EMUL_CONTINUE;
3135 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3139 rc = ctxt->ops->fix_hypercall(ctxt);
3141 /* Disable writeback. */
3142 ctxt->dst.type = OP_NONE;
3146 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3148 struct desc_ptr desc_ptr;
3151 if (ctxt->mode == X86EMUL_MODE_PROT64)
3153 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3154 &desc_ptr.size, &desc_ptr.address,
3156 if (rc != X86EMUL_CONTINUE)
3158 ctxt->ops->set_idt(ctxt, &desc_ptr);
3159 /* Disable writeback. */
3160 ctxt->dst.type = OP_NONE;
3161 return X86EMUL_CONTINUE;
3164 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3166 ctxt->dst.bytes = 2;
3167 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3168 return X86EMUL_CONTINUE;
3171 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3173 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3174 | (ctxt->src.val & 0x0f));
3175 ctxt->dst.type = OP_NONE;
3176 return X86EMUL_CONTINUE;
3179 static int em_loop(struct x86_emulate_ctxt *ctxt)
3181 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3182 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3183 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3184 jmp_rel(ctxt, ctxt->src.val);
3186 return X86EMUL_CONTINUE;
3189 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3191 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3192 jmp_rel(ctxt, ctxt->src.val);
3194 return X86EMUL_CONTINUE;
3197 static int em_in(struct x86_emulate_ctxt *ctxt)
3199 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3201 return X86EMUL_IO_NEEDED;
3203 return X86EMUL_CONTINUE;
3206 static int em_out(struct x86_emulate_ctxt *ctxt)
3208 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3210 /* Disable writeback. */
3211 ctxt->dst.type = OP_NONE;
3212 return X86EMUL_CONTINUE;
3215 static int em_cli(struct x86_emulate_ctxt *ctxt)
3217 if (emulator_bad_iopl(ctxt))
3218 return emulate_gp(ctxt, 0);
3220 ctxt->eflags &= ~X86_EFLAGS_IF;
3221 return X86EMUL_CONTINUE;
3224 static int em_sti(struct x86_emulate_ctxt *ctxt)
3226 if (emulator_bad_iopl(ctxt))
3227 return emulate_gp(ctxt, 0);
3229 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3230 ctxt->eflags |= X86_EFLAGS_IF;
3231 return X86EMUL_CONTINUE;
3234 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3236 u32 eax, ebx, ecx, edx;
3238 eax = reg_read(ctxt, VCPU_REGS_RAX);
3239 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3240 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3241 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3242 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3243 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3244 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3245 return X86EMUL_CONTINUE;
3248 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3250 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3251 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3252 return X86EMUL_CONTINUE;
3255 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3257 switch (ctxt->op_bytes) {
3258 #ifdef CONFIG_X86_64
3260 asm("bswap %0" : "+r"(ctxt->dst.val));
3264 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3267 return X86EMUL_CONTINUE;
3270 static bool valid_cr(int nr)
3282 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3284 if (!valid_cr(ctxt->modrm_reg))
3285 return emulate_ud(ctxt);
3287 return X86EMUL_CONTINUE;
3290 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3292 u64 new_val = ctxt->src.val64;
3293 int cr = ctxt->modrm_reg;
3296 static u64 cr_reserved_bits[] = {
3297 0xffffffff00000000ULL,
3298 0, 0, 0, /* CR3 checked later */
3305 return emulate_ud(ctxt);
3307 if (new_val & cr_reserved_bits[cr])
3308 return emulate_gp(ctxt, 0);
3313 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3314 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3315 return emulate_gp(ctxt, 0);
3317 cr4 = ctxt->ops->get_cr(ctxt, 4);
3318 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3320 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3321 !(cr4 & X86_CR4_PAE))
3322 return emulate_gp(ctxt, 0);
3329 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3330 if (efer & EFER_LMA)
3331 rsvd = CR3_L_MODE_RESERVED_BITS;
3332 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3333 rsvd = CR3_PAE_RESERVED_BITS;
3334 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3335 rsvd = CR3_NONPAE_RESERVED_BITS;
3338 return emulate_gp(ctxt, 0);
3343 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3345 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3346 return emulate_gp(ctxt, 0);
3352 return X86EMUL_CONTINUE;
3355 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3359 ctxt->ops->get_dr(ctxt, 7, &dr7);
3361 /* Check if DR7.Global_Enable is set */
3362 return dr7 & (1 << 13);
3365 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3367 int dr = ctxt->modrm_reg;
3371 return emulate_ud(ctxt);
3373 cr4 = ctxt->ops->get_cr(ctxt, 4);
3374 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3375 return emulate_ud(ctxt);
3377 if (check_dr7_gd(ctxt))
3378 return emulate_db(ctxt);
3380 return X86EMUL_CONTINUE;
3383 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3385 u64 new_val = ctxt->src.val64;
3386 int dr = ctxt->modrm_reg;
3388 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3389 return emulate_gp(ctxt, 0);
3391 return check_dr_read(ctxt);
3394 static int check_svme(struct x86_emulate_ctxt *ctxt)
3398 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3400 if (!(efer & EFER_SVME))
3401 return emulate_ud(ctxt);
3403 return X86EMUL_CONTINUE;
3406 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3408 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3410 /* Valid physical address? */
3411 if (rax & 0xffff000000000000ULL)
3412 return emulate_gp(ctxt, 0);
3414 return check_svme(ctxt);
3417 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3419 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3421 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3422 return emulate_ud(ctxt);
3424 return X86EMUL_CONTINUE;
3427 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3429 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3430 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3432 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3434 return emulate_gp(ctxt, 0);
3436 return X86EMUL_CONTINUE;
3439 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3441 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3442 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3443 return emulate_gp(ctxt, 0);
3445 return X86EMUL_CONTINUE;
3448 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3450 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3451 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3452 return emulate_gp(ctxt, 0);
3454 return X86EMUL_CONTINUE;
3457 #define D(_y) { .flags = (_y) }
3458 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3459 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3460 .check_perm = (_p) }
3461 #define N D(NotImpl)
3462 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3463 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3464 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3465 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3466 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3467 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3468 #define II(_f, _e, _i) \
3469 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3470 #define IIP(_f, _e, _i, _p) \
3471 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3472 .check_perm = (_p) }
3473 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3475 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3476 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3477 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3478 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3479 #define I2bvIP(_f, _e, _i, _p) \
3480 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3482 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3483 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3484 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3486 static const struct opcode group7_rm1[] = {
3487 DI(SrcNone | Priv, monitor),
3488 DI(SrcNone | Priv, mwait),
3492 static const struct opcode group7_rm3[] = {
3493 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3494 II(SrcNone | Prot | VendorSpecific, em_vmmcall, vmmcall),
3495 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3496 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3497 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3498 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3499 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3500 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3503 static const struct opcode group7_rm7[] = {
3505 DIP(SrcNone, rdtscp, check_rdtsc),
3509 static const struct opcode group1[] = {
3511 F(Lock | PageTable, em_or),
3514 F(Lock | PageTable, em_and),
3520 static const struct opcode group1A[] = {
3521 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3524 static const struct opcode group2[] = {
3525 F(DstMem | ModRM, em_rol),
3526 F(DstMem | ModRM, em_ror),
3527 F(DstMem | ModRM, em_rcl),
3528 F(DstMem | ModRM, em_rcr),
3529 F(DstMem | ModRM, em_shl),
3530 F(DstMem | ModRM, em_shr),
3531 F(DstMem | ModRM, em_shl),
3532 F(DstMem | ModRM, em_sar),
3535 static const struct opcode group3[] = {
3536 F(DstMem | SrcImm | NoWrite, em_test),
3537 F(DstMem | SrcImm | NoWrite, em_test),
3538 F(DstMem | SrcNone | Lock, em_not),
3539 F(DstMem | SrcNone | Lock, em_neg),
3540 F(DstXacc | Src2Mem, em_mul_ex),
3541 F(DstXacc | Src2Mem, em_imul_ex),
3542 F(DstXacc | Src2Mem, em_div_ex),
3543 F(DstXacc | Src2Mem, em_idiv_ex),
3546 static const struct opcode group4[] = {
3547 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3548 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3552 static const struct opcode group5[] = {
3553 F(DstMem | SrcNone | Lock, em_inc),
3554 F(DstMem | SrcNone | Lock, em_dec),
3555 I(SrcMem | Stack, em_grp45),
3556 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3557 I(SrcMem | Stack, em_grp45),
3558 I(SrcMemFAddr | ImplicitOps, em_grp45),
3559 I(SrcMem | Stack, em_grp45), D(Undefined),
3562 static const struct opcode group6[] = {
3565 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3566 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3570 static const struct group_dual group7 = { {
3571 II(Mov | DstMem | Priv, em_sgdt, sgdt),
3572 II(Mov | DstMem | Priv, em_sidt, sidt),
3573 II(SrcMem | Priv, em_lgdt, lgdt),
3574 II(SrcMem | Priv, em_lidt, lidt),
3575 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3576 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3577 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3579 I(SrcNone | Priv | VendorSpecific, em_vmcall),
3581 N, EXT(0, group7_rm3),
3582 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3583 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3587 static const struct opcode group8[] = {
3589 F(DstMem | SrcImmByte | NoWrite, em_bt),
3590 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3591 F(DstMem | SrcImmByte | Lock, em_btr),
3592 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3595 static const struct group_dual group9 = { {
3596 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3598 N, N, N, N, N, N, N, N,
3601 static const struct opcode group11[] = {
3602 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3606 static const struct gprefix pfx_0f_6f_0f_7f = {
3607 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3610 static const struct gprefix pfx_vmovntpx = {
3611 I(0, em_mov), N, N, N,
3614 static const struct escape escape_d9 = { {
3615 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3618 N, N, N, N, N, N, N, N,
3620 N, N, N, N, N, N, N, N,
3622 N, N, N, N, N, N, N, N,
3624 N, N, N, N, N, N, N, N,
3626 N, N, N, N, N, N, N, N,
3628 N, N, N, N, N, N, N, N,
3630 N, N, N, N, N, N, N, N,
3632 N, N, N, N, N, N, N, N,
3635 static const struct escape escape_db = { {
3636 N, N, N, N, N, N, N, N,
3639 N, N, N, N, N, N, N, N,
3641 N, N, N, N, N, N, N, N,
3643 N, N, N, N, N, N, N, N,
3645 N, N, N, N, N, N, N, N,
3647 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3649 N, N, N, N, N, N, N, N,
3651 N, N, N, N, N, N, N, N,
3653 N, N, N, N, N, N, N, N,
3656 static const struct escape escape_dd = { {
3657 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3660 N, N, N, N, N, N, N, N,
3662 N, N, N, N, N, N, N, N,
3664 N, N, N, N, N, N, N, N,
3666 N, N, N, N, N, N, N, N,
3668 N, N, N, N, N, N, N, N,
3670 N, N, N, N, N, N, N, N,
3672 N, N, N, N, N, N, N, N,
3674 N, N, N, N, N, N, N, N,
3677 static const struct opcode opcode_table[256] = {
3679 F6ALU(Lock, em_add),
3680 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3681 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3683 F6ALU(Lock | PageTable, em_or),
3684 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3687 F6ALU(Lock, em_adc),
3688 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3689 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3691 F6ALU(Lock, em_sbb),
3692 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3693 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3695 F6ALU(Lock | PageTable, em_and), N, N,
3697 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3699 F6ALU(Lock, em_xor), N, N,
3701 F6ALU(NoWrite, em_cmp), N, N,
3703 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
3705 X8(I(SrcReg | Stack, em_push)),
3707 X8(I(DstReg | Stack, em_pop)),
3709 I(ImplicitOps | Stack | No64, em_pusha),
3710 I(ImplicitOps | Stack | No64, em_popa),
3711 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3714 I(SrcImm | Mov | Stack, em_push),
3715 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3716 I(SrcImmByte | Mov | Stack, em_push),
3717 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3718 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3719 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3723 G(ByteOp | DstMem | SrcImm, group1),
3724 G(DstMem | SrcImm, group1),
3725 G(ByteOp | DstMem | SrcImm | No64, group1),
3726 G(DstMem | SrcImmByte, group1),
3727 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3728 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3730 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3731 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3732 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3733 D(ModRM | SrcMem | NoAccess | DstReg),
3734 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3737 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3739 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3740 I(SrcImmFAddr | No64, em_call_far), N,
3741 II(ImplicitOps | Stack, em_pushf, pushf),
3742 II(ImplicitOps | Stack, em_popf, popf), N, I(ImplicitOps, em_lahf),
3744 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3745 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3746 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3747 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3749 F2bv(DstAcc | SrcImm | NoWrite, em_test),
3750 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3751 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3752 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3754 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3756 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3758 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
3759 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3760 I(ImplicitOps | Stack, em_ret),
3761 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3762 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3763 G(ByteOp, group11), G(0, group11),
3765 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3766 N, I(ImplicitOps | Stack, em_ret_far),
3767 D(ImplicitOps), DI(SrcImmByte, intn),
3768 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3770 G(Src2One | ByteOp, group2), G(Src2One, group2),
3771 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
3772 I(DstAcc | SrcImmUByte | No64, em_aam),
3773 I(DstAcc | SrcImmUByte | No64, em_aad),
3774 F(DstAcc | ByteOp | No64, em_salc),
3775 I(DstAcc | SrcXLat | ByteOp, em_mov),
3777 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3779 X3(I(SrcImmByte, em_loop)),
3780 I(SrcImmByte, em_jcxz),
3781 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3782 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3784 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3785 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3786 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3787 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3789 N, DI(ImplicitOps, icebp), N, N,
3790 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3791 G(ByteOp, group3), G(0, group3),
3793 D(ImplicitOps), D(ImplicitOps),
3794 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3795 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3798 static const struct opcode twobyte_table[256] = {
3800 G(0, group6), GD(0, &group7), N, N,
3801 N, I(ImplicitOps | VendorSpecific, em_syscall),
3802 II(ImplicitOps | Priv, em_clts, clts), N,
3803 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3804 N, D(ImplicitOps | ModRM), N, N,
3806 N, N, N, N, N, N, N, N,
3807 D(ImplicitOps | ModRM), N, N, N, N, N, N, D(ImplicitOps | ModRM),
3809 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3810 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3811 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
3812 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
3814 N, N, N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
3817 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3818 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3819 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3820 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3821 I(ImplicitOps | VendorSpecific, em_sysenter),
3822 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3824 N, N, N, N, N, N, N, N,
3826 X16(D(DstReg | SrcMem | ModRM | Mov)),
3828 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3833 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3838 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3842 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3844 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3845 II(ImplicitOps, em_cpuid, cpuid),
3846 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
3847 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
3848 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
3850 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3851 DI(ImplicitOps, rsm),
3852 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3853 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
3854 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
3855 D(ModRM), F(DstReg | SrcMem | ModRM, em_imul),
3857 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3858 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3859 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3860 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3861 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3862 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3866 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3867 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
3868 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3870 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
3871 N, D(DstMem | SrcReg | ModRM | Mov),
3872 N, N, N, GD(0, &group9),
3874 X8(I(DstReg, em_bswap)),
3876 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3878 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3880 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3897 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3901 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3907 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3908 unsigned size, bool sign_extension)
3910 int rc = X86EMUL_CONTINUE;
3914 op->addr.mem.ea = ctxt->_eip;
3915 /* NB. Immediates are sign-extended as necessary. */
3916 switch (op->bytes) {
3918 op->val = insn_fetch(s8, ctxt);
3921 op->val = insn_fetch(s16, ctxt);
3924 op->val = insn_fetch(s32, ctxt);
3927 op->val = insn_fetch(s64, ctxt);
3930 if (!sign_extension) {
3931 switch (op->bytes) {
3939 op->val &= 0xffffffff;
3947 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
3950 int rc = X86EMUL_CONTINUE;
3954 decode_register_operand(ctxt, op);
3957 rc = decode_imm(ctxt, op, 1, false);
3960 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3964 if ((ctxt->d & BitOp) && op == &ctxt->dst)
3965 fetch_bit_operand(ctxt);
3966 op->orig_val = op->val;
3969 ctxt->memop.bytes = 8;
3973 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3974 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
3975 fetch_register_operand(op);
3976 op->orig_val = op->val;
3980 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
3981 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
3982 fetch_register_operand(op);
3983 op->orig_val = op->val;
3986 if (ctxt->d & ByteOp) {
3991 op->bytes = ctxt->op_bytes;
3992 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3993 fetch_register_operand(op);
3994 op->orig_val = op->val;
3998 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4000 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4001 op->addr.mem.seg = VCPU_SREG_ES;
4008 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4009 fetch_register_operand(op);
4013 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4016 rc = decode_imm(ctxt, op, 1, true);
4023 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4026 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4029 ctxt->memop.bytes = 1;
4030 if (ctxt->memop.type == OP_REG) {
4031 ctxt->memop.addr.reg = decode_register(ctxt, ctxt->modrm_rm, 1);
4032 fetch_register_operand(&ctxt->memop);
4036 ctxt->memop.bytes = 2;
4039 ctxt->memop.bytes = 4;
4042 rc = decode_imm(ctxt, op, 2, false);
4045 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4049 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4051 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4052 op->addr.mem.seg = seg_override(ctxt);
4058 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4060 register_address(ctxt,
4061 reg_read(ctxt, VCPU_REGS_RBX) +
4062 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4063 op->addr.mem.seg = seg_override(ctxt);
4068 op->addr.mem.ea = ctxt->_eip;
4069 op->bytes = ctxt->op_bytes + 2;
4070 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4073 ctxt->memop.bytes = ctxt->op_bytes + 2;
4076 op->val = VCPU_SREG_ES;
4079 op->val = VCPU_SREG_CS;
4082 op->val = VCPU_SREG_SS;
4085 op->val = VCPU_SREG_DS;
4088 op->val = VCPU_SREG_FS;
4091 op->val = VCPU_SREG_GS;
4094 /* Special instructions do their own operand decoding. */
4096 op->type = OP_NONE; /* Disable writeback. */
4104 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4106 int rc = X86EMUL_CONTINUE;
4107 int mode = ctxt->mode;
4108 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4109 bool op_prefix = false;
4110 struct opcode opcode;
4112 ctxt->memop.type = OP_NONE;
4113 ctxt->memopp = NULL;
4114 ctxt->_eip = ctxt->eip;
4115 ctxt->fetch.start = ctxt->_eip;
4116 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4118 memcpy(ctxt->fetch.data, insn, insn_len);
4121 case X86EMUL_MODE_REAL:
4122 case X86EMUL_MODE_VM86:
4123 case X86EMUL_MODE_PROT16:
4124 def_op_bytes = def_ad_bytes = 2;
4126 case X86EMUL_MODE_PROT32:
4127 def_op_bytes = def_ad_bytes = 4;
4129 #ifdef CONFIG_X86_64
4130 case X86EMUL_MODE_PROT64:
4136 return EMULATION_FAILED;
4139 ctxt->op_bytes = def_op_bytes;
4140 ctxt->ad_bytes = def_ad_bytes;
4142 /* Legacy prefixes. */
4144 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4145 case 0x66: /* operand-size override */
4147 /* switch between 2/4 bytes */
4148 ctxt->op_bytes = def_op_bytes ^ 6;
4150 case 0x67: /* address-size override */
4151 if (mode == X86EMUL_MODE_PROT64)
4152 /* switch between 4/8 bytes */
4153 ctxt->ad_bytes = def_ad_bytes ^ 12;
4155 /* switch between 2/4 bytes */
4156 ctxt->ad_bytes = def_ad_bytes ^ 6;
4158 case 0x26: /* ES override */
4159 case 0x2e: /* CS override */
4160 case 0x36: /* SS override */
4161 case 0x3e: /* DS override */
4162 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4164 case 0x64: /* FS override */
4165 case 0x65: /* GS override */
4166 set_seg_override(ctxt, ctxt->b & 7);
4168 case 0x40 ... 0x4f: /* REX */
4169 if (mode != X86EMUL_MODE_PROT64)
4171 ctxt->rex_prefix = ctxt->b;
4173 case 0xf0: /* LOCK */
4174 ctxt->lock_prefix = 1;
4176 case 0xf2: /* REPNE/REPNZ */
4177 case 0xf3: /* REP/REPE/REPZ */
4178 ctxt->rep_prefix = ctxt->b;
4184 /* Any legacy prefix after a REX prefix nullifies its effect. */
4186 ctxt->rex_prefix = 0;
4192 if (ctxt->rex_prefix & 8)
4193 ctxt->op_bytes = 8; /* REX.W */
4195 /* Opcode byte(s). */
4196 opcode = opcode_table[ctxt->b];
4197 /* Two-byte opcode? */
4198 if (ctxt->b == 0x0f) {
4200 ctxt->b = insn_fetch(u8, ctxt);
4201 opcode = twobyte_table[ctxt->b];
4203 ctxt->d = opcode.flags;
4205 if (ctxt->d & ModRM)
4206 ctxt->modrm = insn_fetch(u8, ctxt);
4208 while (ctxt->d & GroupMask) {
4209 switch (ctxt->d & GroupMask) {
4211 goffset = (ctxt->modrm >> 3) & 7;
4212 opcode = opcode.u.group[goffset];
4215 goffset = (ctxt->modrm >> 3) & 7;
4216 if ((ctxt->modrm >> 6) == 3)
4217 opcode = opcode.u.gdual->mod3[goffset];
4219 opcode = opcode.u.gdual->mod012[goffset];
4222 goffset = ctxt->modrm & 7;
4223 opcode = opcode.u.group[goffset];
4226 if (ctxt->rep_prefix && op_prefix)
4227 return EMULATION_FAILED;
4228 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4229 switch (simd_prefix) {
4230 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4231 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4232 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4233 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4237 if (ctxt->modrm > 0xbf)
4238 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4240 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4243 return EMULATION_FAILED;
4246 ctxt->d &= ~(u64)GroupMask;
4247 ctxt->d |= opcode.flags;
4250 ctxt->execute = opcode.u.execute;
4251 ctxt->check_perm = opcode.check_perm;
4252 ctxt->intercept = opcode.intercept;
4255 if (ctxt->d == 0 || (ctxt->d & NotImpl))
4256 return EMULATION_FAILED;
4258 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
4259 return EMULATION_FAILED;
4261 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4264 if (ctxt->d & Op3264) {
4265 if (mode == X86EMUL_MODE_PROT64)
4272 ctxt->op_bytes = 16;
4273 else if (ctxt->d & Mmx)
4276 /* ModRM and SIB bytes. */
4277 if (ctxt->d & ModRM) {
4278 rc = decode_modrm(ctxt, &ctxt->memop);
4279 if (!ctxt->has_seg_override)
4280 set_seg_override(ctxt, ctxt->modrm_seg);
4281 } else if (ctxt->d & MemAbs)
4282 rc = decode_abs(ctxt, &ctxt->memop);
4283 if (rc != X86EMUL_CONTINUE)
4286 if (!ctxt->has_seg_override)
4287 set_seg_override(ctxt, VCPU_SREG_DS);
4289 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4291 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4292 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4295 * Decode and fetch the source operand: register, memory
4298 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4299 if (rc != X86EMUL_CONTINUE)
4303 * Decode and fetch the second source operand: register, memory
4306 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4307 if (rc != X86EMUL_CONTINUE)
4310 /* Decode and fetch the destination operand: register or memory. */
4311 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4314 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4315 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4317 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4320 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4322 return ctxt->d & PageTable;
4325 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4327 /* The second termination condition only applies for REPE
4328 * and REPNE. Test if the repeat string operation prefix is
4329 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4330 * corresponding termination condition according to:
4331 * - if REPE/REPZ and ZF = 0 then done
4332 * - if REPNE/REPNZ and ZF = 1 then done
4334 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4335 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4336 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4337 ((ctxt->eflags & EFLG_ZF) == 0))
4338 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4339 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4345 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4349 ctxt->ops->get_fpu(ctxt);
4350 asm volatile("1: fwait \n\t"
4352 ".pushsection .fixup,\"ax\" \n\t"
4354 "movb $1, %[fault] \n\t"
4357 _ASM_EXTABLE(1b, 3b)
4358 : [fault]"+qm"(fault));
4359 ctxt->ops->put_fpu(ctxt);
4361 if (unlikely(fault))
4362 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4364 return X86EMUL_CONTINUE;
4367 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4370 if (op->type == OP_MM)
4371 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4374 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4376 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4377 if (!(ctxt->d & ByteOp))
4378 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4379 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4380 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
4382 : "c"(ctxt->src2.val));
4383 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4384 if (!fop) /* exception is returned in fop variable */
4385 return emulate_de(ctxt);
4386 return X86EMUL_CONTINUE;
4389 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4391 const struct x86_emulate_ops *ops = ctxt->ops;
4392 int rc = X86EMUL_CONTINUE;
4393 int saved_dst_type = ctxt->dst.type;
4395 ctxt->mem_read.pos = 0;
4397 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4398 (ctxt->d & Undefined)) {
4399 rc = emulate_ud(ctxt);
4403 /* LOCK prefix is allowed only with some instructions */
4404 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4405 rc = emulate_ud(ctxt);
4409 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4410 rc = emulate_ud(ctxt);
4414 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4415 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4416 rc = emulate_ud(ctxt);
4420 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4421 rc = emulate_nm(ctxt);
4425 if (ctxt->d & Mmx) {
4426 rc = flush_pending_x87_faults(ctxt);
4427 if (rc != X86EMUL_CONTINUE)
4430 * Now that we know the fpu is exception safe, we can fetch
4433 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4434 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4435 if (!(ctxt->d & Mov))
4436 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4439 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4440 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4441 X86_ICPT_PRE_EXCEPT);
4442 if (rc != X86EMUL_CONTINUE)
4446 /* Privileged instruction can be executed only in CPL=0 */
4447 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4448 rc = emulate_gp(ctxt, 0);
4452 /* Instruction can only be executed in protected mode */
4453 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4454 rc = emulate_ud(ctxt);
4458 /* Do instruction specific permission checks */
4459 if (ctxt->check_perm) {
4460 rc = ctxt->check_perm(ctxt);
4461 if (rc != X86EMUL_CONTINUE)
4465 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4466 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4467 X86_ICPT_POST_EXCEPT);
4468 if (rc != X86EMUL_CONTINUE)
4472 if (ctxt->rep_prefix && (ctxt->d & String)) {
4473 /* All REP prefixes have the same first termination condition */
4474 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4475 ctxt->eip = ctxt->_eip;
4480 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4481 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4482 ctxt->src.valptr, ctxt->src.bytes);
4483 if (rc != X86EMUL_CONTINUE)
4485 ctxt->src.orig_val64 = ctxt->src.val64;
4488 if (ctxt->src2.type == OP_MEM) {
4489 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4490 &ctxt->src2.val, ctxt->src2.bytes);
4491 if (rc != X86EMUL_CONTINUE)
4495 if ((ctxt->d & DstMask) == ImplicitOps)
4499 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4500 /* optimisation - avoid slow emulated read if Mov */
4501 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4502 &ctxt->dst.val, ctxt->dst.bytes);
4503 if (rc != X86EMUL_CONTINUE)
4506 ctxt->dst.orig_val = ctxt->dst.val;
4510 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4511 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4512 X86_ICPT_POST_MEMACCESS);
4513 if (rc != X86EMUL_CONTINUE)
4517 if (ctxt->execute) {
4518 if (ctxt->d & Fastop) {
4519 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4520 rc = fastop(ctxt, fop);
4521 if (rc != X86EMUL_CONTINUE)
4525 rc = ctxt->execute(ctxt);
4526 if (rc != X86EMUL_CONTINUE)
4535 case 0x63: /* movsxd */
4536 if (ctxt->mode != X86EMUL_MODE_PROT64)
4537 goto cannot_emulate;
4538 ctxt->dst.val = (s32) ctxt->src.val;
4540 case 0x70 ... 0x7f: /* jcc (short) */
4541 if (test_cc(ctxt->b, ctxt->eflags))
4542 jmp_rel(ctxt, ctxt->src.val);
4544 case 0x8d: /* lea r16/r32, m */
4545 ctxt->dst.val = ctxt->src.addr.mem.ea;
4547 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4548 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4552 case 0x98: /* cbw/cwde/cdqe */
4553 switch (ctxt->op_bytes) {
4554 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4555 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4556 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4559 case 0xcc: /* int3 */
4560 rc = emulate_int(ctxt, 3);
4562 case 0xcd: /* int n */
4563 rc = emulate_int(ctxt, ctxt->src.val);
4565 case 0xce: /* into */
4566 if (ctxt->eflags & EFLG_OF)
4567 rc = emulate_int(ctxt, 4);
4569 case 0xe9: /* jmp rel */
4570 case 0xeb: /* jmp rel short */
4571 jmp_rel(ctxt, ctxt->src.val);
4572 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4574 case 0xf4: /* hlt */
4575 ctxt->ops->halt(ctxt);
4577 case 0xf5: /* cmc */
4578 /* complement carry flag from eflags reg */
4579 ctxt->eflags ^= EFLG_CF;
4581 case 0xf8: /* clc */
4582 ctxt->eflags &= ~EFLG_CF;
4584 case 0xf9: /* stc */
4585 ctxt->eflags |= EFLG_CF;
4587 case 0xfc: /* cld */
4588 ctxt->eflags &= ~EFLG_DF;
4590 case 0xfd: /* std */
4591 ctxt->eflags |= EFLG_DF;
4594 goto cannot_emulate;
4597 if (rc != X86EMUL_CONTINUE)
4601 if (!(ctxt->d & NoWrite)) {
4602 rc = writeback(ctxt, &ctxt->dst);
4603 if (rc != X86EMUL_CONTINUE)
4606 if (ctxt->d & SrcWrite) {
4607 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
4608 rc = writeback(ctxt, &ctxt->src);
4609 if (rc != X86EMUL_CONTINUE)
4614 * restore dst type in case the decoding will be reused
4615 * (happens for string instruction )
4617 ctxt->dst.type = saved_dst_type;
4619 if ((ctxt->d & SrcMask) == SrcSI)
4620 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4622 if ((ctxt->d & DstMask) == DstDI)
4623 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4625 if (ctxt->rep_prefix && (ctxt->d & String)) {
4627 struct read_cache *r = &ctxt->io_read;
4628 if ((ctxt->d & SrcMask) == SrcSI)
4629 count = ctxt->src.count;
4631 count = ctxt->dst.count;
4632 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4635 if (!string_insn_completed(ctxt)) {
4637 * Re-enter guest when pio read ahead buffer is empty
4638 * or, if it is not used, after each 1024 iteration.
4640 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4641 (r->end == 0 || r->end != r->pos)) {
4643 * Reset read cache. Usually happens before
4644 * decode, but since instruction is restarted
4645 * we have to do it here.
4647 ctxt->mem_read.end = 0;
4648 writeback_registers(ctxt);
4649 return EMULATION_RESTART;
4651 goto done; /* skip rip writeback */
4655 ctxt->eip = ctxt->_eip;
4658 if (rc == X86EMUL_PROPAGATE_FAULT)
4659 ctxt->have_exception = true;
4660 if (rc == X86EMUL_INTERCEPTED)
4661 return EMULATION_INTERCEPTED;
4663 if (rc == X86EMUL_CONTINUE)
4664 writeback_registers(ctxt);
4666 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4670 case 0x09: /* wbinvd */
4671 (ctxt->ops->wbinvd)(ctxt);
4673 case 0x08: /* invd */
4674 case 0x0d: /* GrpP (prefetch) */
4675 case 0x18: /* Grp16 (prefetch/nop) */
4676 case 0x1f: /* nop */
4678 case 0x20: /* mov cr, reg */
4679 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4681 case 0x21: /* mov from dr to reg */
4682 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4684 case 0x40 ... 0x4f: /* cmov */
4685 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4686 if (!test_cc(ctxt->b, ctxt->eflags))
4687 ctxt->dst.type = OP_NONE; /* no writeback */
4689 case 0x80 ... 0x8f: /* jnz rel, etc*/
4690 if (test_cc(ctxt->b, ctxt->eflags))
4691 jmp_rel(ctxt, ctxt->src.val);
4693 case 0x90 ... 0x9f: /* setcc r/m8 */
4694 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4696 case 0xae: /* clflush */
4698 case 0xb6 ... 0xb7: /* movzx */
4699 ctxt->dst.bytes = ctxt->op_bytes;
4700 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4701 : (u16) ctxt->src.val;
4703 case 0xbe ... 0xbf: /* movsx */
4704 ctxt->dst.bytes = ctxt->op_bytes;
4705 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4706 (s16) ctxt->src.val;
4708 case 0xc3: /* movnti */
4709 ctxt->dst.bytes = ctxt->op_bytes;
4710 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4711 (u64) ctxt->src.val;
4714 goto cannot_emulate;
4717 if (rc != X86EMUL_CONTINUE)
4723 return EMULATION_FAILED;
4726 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4728 invalidate_registers(ctxt);
4731 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4733 writeback_registers(ctxt);