1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstImmUByte (OpImmUByte << DstShift)
90 #define DstDX (OpDX << DstShift)
91 #define DstAccLo (OpAccLo << DstShift)
92 #define DstMask (OpMask << DstShift)
93 /* Source operand type. */
95 #define SrcNone (OpNone << SrcShift)
96 #define SrcReg (OpReg << SrcShift)
97 #define SrcMem (OpMem << SrcShift)
98 #define SrcMem16 (OpMem16 << SrcShift)
99 #define SrcMem32 (OpMem32 << SrcShift)
100 #define SrcImm (OpImm << SrcShift)
101 #define SrcImmByte (OpImmByte << SrcShift)
102 #define SrcOne (OpOne << SrcShift)
103 #define SrcImmUByte (OpImmUByte << SrcShift)
104 #define SrcImmU (OpImmU << SrcShift)
105 #define SrcSI (OpSI << SrcShift)
106 #define SrcXLat (OpXLat << SrcShift)
107 #define SrcImmFAddr (OpImmFAddr << SrcShift)
108 #define SrcMemFAddr (OpMemFAddr << SrcShift)
109 #define SrcAcc (OpAcc << SrcShift)
110 #define SrcImmU16 (OpImmU16 << SrcShift)
111 #define SrcImm64 (OpImm64 << SrcShift)
112 #define SrcDX (OpDX << SrcShift)
113 #define SrcMem8 (OpMem8 << SrcShift)
114 #define SrcAccHi (OpAccHi << SrcShift)
115 #define SrcMask (OpMask << SrcShift)
116 #define BitOp (1<<11)
117 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
118 #define String (1<<13) /* String instruction (rep capable) */
119 #define Stack (1<<14) /* Stack instruction (push/pop) */
120 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
121 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
122 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
123 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
124 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
125 #define Escape (5<<15) /* Escape to coprocessor instruction */
126 #define Sse (1<<18) /* SSE Vector instruction */
127 /* Generic ModRM decode. */
128 #define ModRM (1<<19)
129 /* Destination is only written; never read. */
132 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
133 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
134 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
135 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
136 #define Undefined (1<<25) /* No Such Instruction */
137 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
138 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
140 #define PageTable (1 << 29) /* instruction used to write page table */
141 #define NotImpl (1 << 30) /* instruction is not implemented */
142 /* Source 2 operand type */
143 #define Src2Shift (31)
144 #define Src2None (OpNone << Src2Shift)
145 #define Src2Mem (OpMem << Src2Shift)
146 #define Src2CL (OpCL << Src2Shift)
147 #define Src2ImmByte (OpImmByte << Src2Shift)
148 #define Src2One (OpOne << Src2Shift)
149 #define Src2Imm (OpImm << Src2Shift)
150 #define Src2ES (OpES << Src2Shift)
151 #define Src2CS (OpCS << Src2Shift)
152 #define Src2SS (OpSS << Src2Shift)
153 #define Src2DS (OpDS << Src2Shift)
154 #define Src2FS (OpFS << Src2Shift)
155 #define Src2GS (OpGS << Src2Shift)
156 #define Src2Mask (OpMask << Src2Shift)
157 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
158 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
159 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
160 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
161 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
162 #define NoWrite ((u64)1 << 45) /* No writeback */
163 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
164 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
165 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
166 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
167 #define NoBigReal ((u64)1 << 50) /* No big real mode */
168 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
170 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
172 #define X2(x...) x, x
173 #define X3(x...) X2(x), x
174 #define X4(x...) X2(x), X2(x)
175 #define X5(x...) X4(x), x
176 #define X6(x...) X4(x), X2(x)
177 #define X7(x...) X4(x), X3(x)
178 #define X8(x...) X4(x), X4(x)
179 #define X16(x...) X8(x), X8(x)
181 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
182 #define FASTOP_SIZE 8
185 * fastop functions have a special calling convention:
190 * flags: rflags (in/out)
191 * ex: rsi (in:fastop pointer, out:zero if exception)
193 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
194 * different operand sizes can be reached by calculation, rather than a jump
195 * table (which would be bigger than the code).
197 * fastop functions are declared as taking a never-defined fastop parameter,
198 * so they can't be called from C directly.
207 int (*execute)(struct x86_emulate_ctxt *ctxt);
208 const struct opcode *group;
209 const struct group_dual *gdual;
210 const struct gprefix *gprefix;
211 const struct escape *esc;
212 void (*fastop)(struct fastop *fake);
214 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
218 struct opcode mod012[8];
219 struct opcode mod3[8];
223 struct opcode pfx_no;
224 struct opcode pfx_66;
225 struct opcode pfx_f2;
226 struct opcode pfx_f3;
231 struct opcode high[64];
234 /* EFLAGS bit definitions. */
235 #define EFLG_ID (1<<21)
236 #define EFLG_VIP (1<<20)
237 #define EFLG_VIF (1<<19)
238 #define EFLG_AC (1<<18)
239 #define EFLG_VM (1<<17)
240 #define EFLG_RF (1<<16)
241 #define EFLG_IOPL (3<<12)
242 #define EFLG_NT (1<<14)
243 #define EFLG_OF (1<<11)
244 #define EFLG_DF (1<<10)
245 #define EFLG_IF (1<<9)
246 #define EFLG_TF (1<<8)
247 #define EFLG_SF (1<<7)
248 #define EFLG_ZF (1<<6)
249 #define EFLG_AF (1<<4)
250 #define EFLG_PF (1<<2)
251 #define EFLG_CF (1<<0)
253 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
254 #define EFLG_RESERVED_ONE_MASK 2
256 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
258 if (!(ctxt->regs_valid & (1 << nr))) {
259 ctxt->regs_valid |= 1 << nr;
260 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
262 return ctxt->_regs[nr];
265 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
267 ctxt->regs_valid |= 1 << nr;
268 ctxt->regs_dirty |= 1 << nr;
269 return &ctxt->_regs[nr];
272 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
275 return reg_write(ctxt, nr);
278 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
282 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
283 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
286 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
288 ctxt->regs_dirty = 0;
289 ctxt->regs_valid = 0;
293 * These EFLAGS bits are restored from saved value during emulation, and
294 * any changes are written back to the saved value after emulation.
296 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
304 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
306 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
307 #define FOP_RET "ret \n\t"
309 #define FOP_START(op) \
310 extern void em_##op(struct fastop *fake); \
311 asm(".pushsection .text, \"ax\" \n\t" \
312 ".global em_" #op " \n\t" \
319 #define FOPNOP() FOP_ALIGN FOP_RET
321 #define FOP1E(op, dst) \
322 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
324 #define FOP1EEX(op, dst) \
325 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
327 #define FASTOP1(op) \
332 ON64(FOP1E(op##q, rax)) \
335 /* 1-operand, using src2 (for MUL/DIV r/m) */
336 #define FASTOP1SRC2(op, name) \
341 ON64(FOP1E(op, rcx)) \
344 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
345 #define FASTOP1SRC2EX(op, name) \
350 ON64(FOP1EEX(op, rcx)) \
353 #define FOP2E(op, dst, src) \
354 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
356 #define FASTOP2(op) \
358 FOP2E(op##b, al, dl) \
359 FOP2E(op##w, ax, dx) \
360 FOP2E(op##l, eax, edx) \
361 ON64(FOP2E(op##q, rax, rdx)) \
364 /* 2 operand, word only */
365 #define FASTOP2W(op) \
368 FOP2E(op##w, ax, dx) \
369 FOP2E(op##l, eax, edx) \
370 ON64(FOP2E(op##q, rax, rdx)) \
373 /* 2 operand, src is CL */
374 #define FASTOP2CL(op) \
376 FOP2E(op##b, al, cl) \
377 FOP2E(op##w, ax, cl) \
378 FOP2E(op##l, eax, cl) \
379 ON64(FOP2E(op##q, rax, cl)) \
382 #define FOP3E(op, dst, src, src2) \
383 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
385 /* 3-operand, word-only, src2=cl */
386 #define FASTOP3WCL(op) \
389 FOP3E(op##w, ax, dx, cl) \
390 FOP3E(op##l, eax, edx, cl) \
391 ON64(FOP3E(op##q, rax, rdx, cl)) \
394 /* Special case for SETcc - 1 instruction per cc */
395 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
397 asm(".global kvm_fastop_exception \n"
398 "kvm_fastop_exception: xor %esi, %esi; ret");
419 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
422 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
423 enum x86_intercept intercept,
424 enum x86_intercept_stage stage)
426 struct x86_instruction_info info = {
427 .intercept = intercept,
428 .rep_prefix = ctxt->rep_prefix,
429 .modrm_mod = ctxt->modrm_mod,
430 .modrm_reg = ctxt->modrm_reg,
431 .modrm_rm = ctxt->modrm_rm,
432 .src_val = ctxt->src.val64,
433 .dst_val = ctxt->dst.val64,
434 .src_bytes = ctxt->src.bytes,
435 .dst_bytes = ctxt->dst.bytes,
436 .ad_bytes = ctxt->ad_bytes,
437 .next_rip = ctxt->eip,
440 return ctxt->ops->intercept(ctxt, &info, stage);
443 static void assign_masked(ulong *dest, ulong src, ulong mask)
445 *dest = (*dest & ~mask) | (src & mask);
448 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
450 return (1UL << (ctxt->ad_bytes << 3)) - 1;
453 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
456 struct desc_struct ss;
458 if (ctxt->mode == X86EMUL_MODE_PROT64)
460 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
461 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
464 static int stack_size(struct x86_emulate_ctxt *ctxt)
466 return (__fls(stack_mask(ctxt)) + 1) >> 3;
469 /* Access/update address held in a register, based on addressing mode. */
470 static inline unsigned long
471 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
473 if (ctxt->ad_bytes == sizeof(unsigned long))
476 return reg & ad_mask(ctxt);
479 static inline unsigned long
480 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
482 return address_mask(ctxt, reg);
485 static void masked_increment(ulong *reg, ulong mask, int inc)
487 assign_masked(reg, *reg + inc, mask);
491 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
495 if (ctxt->ad_bytes == sizeof(unsigned long))
498 mask = ad_mask(ctxt);
499 masked_increment(reg, mask, inc);
502 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
504 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
507 static u32 desc_limit_scaled(struct desc_struct *desc)
509 u32 limit = get_desc_limit(desc);
511 return desc->g ? (limit << 12) | 0xfff : limit;
514 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
516 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
519 return ctxt->ops->get_cached_segment_base(ctxt, seg);
522 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
523 u32 error, bool valid)
526 ctxt->exception.vector = vec;
527 ctxt->exception.error_code = error;
528 ctxt->exception.error_code_valid = valid;
529 return X86EMUL_PROPAGATE_FAULT;
532 static int emulate_db(struct x86_emulate_ctxt *ctxt)
534 return emulate_exception(ctxt, DB_VECTOR, 0, false);
537 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
539 return emulate_exception(ctxt, GP_VECTOR, err, true);
542 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
544 return emulate_exception(ctxt, SS_VECTOR, err, true);
547 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
549 return emulate_exception(ctxt, UD_VECTOR, 0, false);
552 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
554 return emulate_exception(ctxt, TS_VECTOR, err, true);
557 static int emulate_de(struct x86_emulate_ctxt *ctxt)
559 return emulate_exception(ctxt, DE_VECTOR, 0, false);
562 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
564 return emulate_exception(ctxt, NM_VECTOR, 0, false);
567 static inline void assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
569 switch (ctxt->op_bytes) {
571 ctxt->_eip = (u16)dst;
574 ctxt->_eip = (u32)dst;
580 WARN(1, "unsupported eip assignment size\n");
584 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
586 assign_eip_near(ctxt, ctxt->_eip + rel);
589 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
592 struct desc_struct desc;
594 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
598 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
603 struct desc_struct desc;
605 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
606 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
610 * x86 defines three classes of vector instructions: explicitly
611 * aligned, explicitly unaligned, and the rest, which change behaviour
612 * depending on whether they're AVX encoded or not.
614 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
615 * subject to the same check.
617 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
619 if (likely(size < 16))
622 if (ctxt->d & Aligned)
624 else if (ctxt->d & Unaligned)
626 else if (ctxt->d & Avx)
632 static int __linearize(struct x86_emulate_ctxt *ctxt,
633 struct segmented_address addr,
634 unsigned size, bool write, bool fetch,
637 struct desc_struct desc;
644 la = seg_base(ctxt, addr.seg) + addr.ea;
645 switch (ctxt->mode) {
646 case X86EMUL_MODE_PROT64:
647 if (((signed long)la << 16) >> 16 != la)
648 return emulate_gp(ctxt, 0);
651 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
655 /* code segment in protected mode or read-only data segment */
656 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
657 || !(desc.type & 2)) && write)
659 /* unreadable code segment */
660 if (!fetch && (desc.type & 8) && !(desc.type & 2))
662 lim = desc_limit_scaled(&desc);
663 if ((ctxt->mode == X86EMUL_MODE_REAL) && !fetch &&
664 (ctxt->d & NoBigReal)) {
665 /* la is between zero and 0xffff */
666 if (la > 0xffff || (u32)(la + size - 1) > 0xffff)
668 } else if ((desc.type & 8) || !(desc.type & 4)) {
669 /* expand-up segment */
670 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
673 /* expand-down segment */
674 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
676 lim = desc.d ? 0xffffffff : 0xffff;
677 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
680 cpl = ctxt->ops->cpl(ctxt);
681 if (!(desc.type & 8)) {
685 } else if ((desc.type & 8) && !(desc.type & 4)) {
686 /* nonconforming code segment */
689 } else if ((desc.type & 8) && (desc.type & 4)) {
690 /* conforming code segment */
696 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
698 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
699 return emulate_gp(ctxt, 0);
701 return X86EMUL_CONTINUE;
703 if (addr.seg == VCPU_SREG_SS)
704 return emulate_ss(ctxt, sel);
706 return emulate_gp(ctxt, sel);
709 static int linearize(struct x86_emulate_ctxt *ctxt,
710 struct segmented_address addr,
711 unsigned size, bool write,
714 return __linearize(ctxt, addr, size, write, false, linear);
718 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
719 struct segmented_address addr,
726 rc = linearize(ctxt, addr, size, false, &linear);
727 if (rc != X86EMUL_CONTINUE)
729 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
733 * Prefetch the remaining bytes of the instruction without crossing page
734 * boundary if they are not in fetch_cache yet.
736 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
740 unsigned long linear;
741 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
742 struct segmented_address addr = { .seg = VCPU_SREG_CS,
743 .ea = ctxt->eip + cur_size };
745 size = 15UL ^ cur_size;
746 rc = __linearize(ctxt, addr, size, false, true, &linear);
747 if (unlikely(rc != X86EMUL_CONTINUE))
750 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
753 * One instruction can only straddle two pages,
754 * and one has been loaded at the beginning of
755 * x86_decode_insn. So, if not enough bytes
756 * still, we must have hit the 15-byte boundary.
758 if (unlikely(size < op_size))
759 return X86EMUL_UNHANDLEABLE;
760 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
761 size, &ctxt->exception);
762 if (unlikely(rc != X86EMUL_CONTINUE))
764 ctxt->fetch.end += size;
765 return X86EMUL_CONTINUE;
768 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
771 if (unlikely(ctxt->fetch.end - ctxt->fetch.ptr < size))
772 return __do_insn_fetch_bytes(ctxt, size);
774 return X86EMUL_CONTINUE;
777 /* Fetch next part of the instruction being emulated. */
778 #define insn_fetch(_type, _ctxt) \
781 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
782 if (rc != X86EMUL_CONTINUE) \
784 ctxt->_eip += sizeof(_type); \
785 _x = *(_type __aligned(1) *) ctxt->fetch.ptr; \
786 ctxt->fetch.ptr += sizeof(_type); \
790 #define insn_fetch_arr(_arr, _size, _ctxt) \
792 rc = do_insn_fetch_bytes(_ctxt, _size); \
793 if (rc != X86EMUL_CONTINUE) \
795 ctxt->_eip += (_size); \
796 memcpy(_arr, ctxt->fetch.ptr, _size); \
797 ctxt->fetch.ptr += (_size); \
801 * Given the 'reg' portion of a ModRM byte, and a register block, return a
802 * pointer into the block that addresses the relevant register.
803 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
805 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
809 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
811 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
812 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
814 p = reg_rmw(ctxt, modrm_reg);
818 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
819 struct segmented_address addr,
820 u16 *size, unsigned long *address, int op_bytes)
827 rc = segmented_read_std(ctxt, addr, size, 2);
828 if (rc != X86EMUL_CONTINUE)
831 rc = segmented_read_std(ctxt, addr, address, op_bytes);
845 FASTOP1SRC2(mul, mul_ex);
846 FASTOP1SRC2(imul, imul_ex);
847 FASTOP1SRC2EX(div, div_ex);
848 FASTOP1SRC2EX(idiv, idiv_ex);
877 static u8 test_cc(unsigned int condition, unsigned long flags)
880 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
882 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
883 asm("push %[flags]; popf; call *%[fastop]"
884 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
888 static void fetch_register_operand(struct operand *op)
892 op->val = *(u8 *)op->addr.reg;
895 op->val = *(u16 *)op->addr.reg;
898 op->val = *(u32 *)op->addr.reg;
901 op->val = *(u64 *)op->addr.reg;
906 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
908 ctxt->ops->get_fpu(ctxt);
910 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
911 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
912 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
913 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
914 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
915 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
916 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
917 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
919 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
920 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
921 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
922 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
923 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
924 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
925 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
926 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
930 ctxt->ops->put_fpu(ctxt);
933 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
936 ctxt->ops->get_fpu(ctxt);
938 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
939 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
940 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
941 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
942 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
943 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
944 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
945 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
947 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
948 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
949 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
950 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
951 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
952 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
953 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
954 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
958 ctxt->ops->put_fpu(ctxt);
961 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
963 ctxt->ops->get_fpu(ctxt);
965 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
966 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
967 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
968 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
969 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
970 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
971 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
972 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
975 ctxt->ops->put_fpu(ctxt);
978 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
980 ctxt->ops->get_fpu(ctxt);
982 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
983 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
984 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
985 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
986 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
987 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
988 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
989 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
992 ctxt->ops->put_fpu(ctxt);
995 static int em_fninit(struct x86_emulate_ctxt *ctxt)
997 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
998 return emulate_nm(ctxt);
1000 ctxt->ops->get_fpu(ctxt);
1001 asm volatile("fninit");
1002 ctxt->ops->put_fpu(ctxt);
1003 return X86EMUL_CONTINUE;
1006 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1010 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1011 return emulate_nm(ctxt);
1013 ctxt->ops->get_fpu(ctxt);
1014 asm volatile("fnstcw %0": "+m"(fcw));
1015 ctxt->ops->put_fpu(ctxt);
1017 /* force 2 byte destination */
1018 ctxt->dst.bytes = 2;
1019 ctxt->dst.val = fcw;
1021 return X86EMUL_CONTINUE;
1024 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1028 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1029 return emulate_nm(ctxt);
1031 ctxt->ops->get_fpu(ctxt);
1032 asm volatile("fnstsw %0": "+m"(fsw));
1033 ctxt->ops->put_fpu(ctxt);
1035 /* force 2 byte destination */
1036 ctxt->dst.bytes = 2;
1037 ctxt->dst.val = fsw;
1039 return X86EMUL_CONTINUE;
1042 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1045 unsigned reg = ctxt->modrm_reg;
1047 if (!(ctxt->d & ModRM))
1048 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1050 if (ctxt->d & Sse) {
1054 read_sse_reg(ctxt, &op->vec_val, reg);
1057 if (ctxt->d & Mmx) {
1066 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1067 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1069 fetch_register_operand(op);
1070 op->orig_val = op->val;
1073 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1075 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1076 ctxt->modrm_seg = VCPU_SREG_SS;
1079 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1083 int index_reg, base_reg, scale;
1084 int rc = X86EMUL_CONTINUE;
1087 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1088 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1089 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1091 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1092 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1093 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1094 ctxt->modrm_seg = VCPU_SREG_DS;
1096 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1098 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1099 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1101 if (ctxt->d & Sse) {
1104 op->addr.xmm = ctxt->modrm_rm;
1105 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1108 if (ctxt->d & Mmx) {
1111 op->addr.mm = ctxt->modrm_rm & 7;
1114 fetch_register_operand(op);
1120 if (ctxt->ad_bytes == 2) {
1121 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1122 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1123 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1124 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1126 /* 16-bit ModR/M decode. */
1127 switch (ctxt->modrm_mod) {
1129 if (ctxt->modrm_rm == 6)
1130 modrm_ea += insn_fetch(u16, ctxt);
1133 modrm_ea += insn_fetch(s8, ctxt);
1136 modrm_ea += insn_fetch(u16, ctxt);
1139 switch (ctxt->modrm_rm) {
1141 modrm_ea += bx + si;
1144 modrm_ea += bx + di;
1147 modrm_ea += bp + si;
1150 modrm_ea += bp + di;
1159 if (ctxt->modrm_mod != 0)
1166 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1167 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1168 ctxt->modrm_seg = VCPU_SREG_SS;
1169 modrm_ea = (u16)modrm_ea;
1171 /* 32/64-bit ModR/M decode. */
1172 if ((ctxt->modrm_rm & 7) == 4) {
1173 sib = insn_fetch(u8, ctxt);
1174 index_reg |= (sib >> 3) & 7;
1175 base_reg |= sib & 7;
1178 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1179 modrm_ea += insn_fetch(s32, ctxt);
1181 modrm_ea += reg_read(ctxt, base_reg);
1182 adjust_modrm_seg(ctxt, base_reg);
1185 modrm_ea += reg_read(ctxt, index_reg) << scale;
1186 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1187 if (ctxt->mode == X86EMUL_MODE_PROT64)
1188 ctxt->rip_relative = 1;
1190 base_reg = ctxt->modrm_rm;
1191 modrm_ea += reg_read(ctxt, base_reg);
1192 adjust_modrm_seg(ctxt, base_reg);
1194 switch (ctxt->modrm_mod) {
1196 if (ctxt->modrm_rm == 5)
1197 modrm_ea += insn_fetch(s32, ctxt);
1200 modrm_ea += insn_fetch(s8, ctxt);
1203 modrm_ea += insn_fetch(s32, ctxt);
1207 op->addr.mem.ea = modrm_ea;
1208 if (ctxt->ad_bytes != 8)
1209 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1215 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1218 int rc = X86EMUL_CONTINUE;
1221 switch (ctxt->ad_bytes) {
1223 op->addr.mem.ea = insn_fetch(u16, ctxt);
1226 op->addr.mem.ea = insn_fetch(u32, ctxt);
1229 op->addr.mem.ea = insn_fetch(u64, ctxt);
1236 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1240 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1241 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1243 if (ctxt->src.bytes == 2)
1244 sv = (s16)ctxt->src.val & (s16)mask;
1245 else if (ctxt->src.bytes == 4)
1246 sv = (s32)ctxt->src.val & (s32)mask;
1248 sv = (s64)ctxt->src.val & (s64)mask;
1250 ctxt->dst.addr.mem.ea += (sv >> 3);
1253 /* only subword offset */
1254 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1257 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1258 unsigned long addr, void *dest, unsigned size)
1261 struct read_cache *mc = &ctxt->mem_read;
1263 if (mc->pos < mc->end)
1266 WARN_ON((mc->end + size) >= sizeof(mc->data));
1268 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1270 if (rc != X86EMUL_CONTINUE)
1276 memcpy(dest, mc->data + mc->pos, size);
1278 return X86EMUL_CONTINUE;
1281 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1282 struct segmented_address addr,
1289 rc = linearize(ctxt, addr, size, false, &linear);
1290 if (rc != X86EMUL_CONTINUE)
1292 return read_emulated(ctxt, linear, data, size);
1295 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1296 struct segmented_address addr,
1303 rc = linearize(ctxt, addr, size, true, &linear);
1304 if (rc != X86EMUL_CONTINUE)
1306 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1310 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1311 struct segmented_address addr,
1312 const void *orig_data, const void *data,
1318 rc = linearize(ctxt, addr, size, true, &linear);
1319 if (rc != X86EMUL_CONTINUE)
1321 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1322 size, &ctxt->exception);
1325 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1326 unsigned int size, unsigned short port,
1329 struct read_cache *rc = &ctxt->io_read;
1331 if (rc->pos == rc->end) { /* refill pio read ahead */
1332 unsigned int in_page, n;
1333 unsigned int count = ctxt->rep_prefix ?
1334 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1335 in_page = (ctxt->eflags & EFLG_DF) ?
1336 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1337 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1338 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1341 rc->pos = rc->end = 0;
1342 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1347 if (ctxt->rep_prefix && (ctxt->d & String) &&
1348 !(ctxt->eflags & EFLG_DF)) {
1349 ctxt->dst.data = rc->data + rc->pos;
1350 ctxt->dst.type = OP_MEM_STR;
1351 ctxt->dst.count = (rc->end - rc->pos) / size;
1354 memcpy(dest, rc->data + rc->pos, size);
1360 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1361 u16 index, struct desc_struct *desc)
1366 ctxt->ops->get_idt(ctxt, &dt);
1368 if (dt.size < index * 8 + 7)
1369 return emulate_gp(ctxt, index << 3 | 0x2);
1371 addr = dt.address + index * 8;
1372 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1376 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1377 u16 selector, struct desc_ptr *dt)
1379 const struct x86_emulate_ops *ops = ctxt->ops;
1382 if (selector & 1 << 2) {
1383 struct desc_struct desc;
1386 memset (dt, 0, sizeof *dt);
1387 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1391 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1392 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1394 ops->get_gdt(ctxt, dt);
1397 /* allowed just for 8 bytes segments */
1398 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1399 u16 selector, struct desc_struct *desc,
1403 u16 index = selector >> 3;
1406 get_descriptor_table_ptr(ctxt, selector, &dt);
1408 if (dt.size < index * 8 + 7)
1409 return emulate_gp(ctxt, selector & 0xfffc);
1411 *desc_addr_p = addr = dt.address + index * 8;
1412 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1416 /* allowed just for 8 bytes segments */
1417 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1418 u16 selector, struct desc_struct *desc)
1421 u16 index = selector >> 3;
1424 get_descriptor_table_ptr(ctxt, selector, &dt);
1426 if (dt.size < index * 8 + 7)
1427 return emulate_gp(ctxt, selector & 0xfffc);
1429 addr = dt.address + index * 8;
1430 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1434 /* Does not support long mode */
1435 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1436 u16 selector, int seg, u8 cpl, bool in_task_switch)
1438 struct desc_struct seg_desc, old_desc;
1440 unsigned err_vec = GP_VECTOR;
1442 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1448 memset(&seg_desc, 0, sizeof seg_desc);
1450 if (ctxt->mode == X86EMUL_MODE_REAL) {
1451 /* set real mode segment descriptor (keep limit etc. for
1453 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1454 set_desc_base(&seg_desc, selector << 4);
1456 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1457 /* VM86 needs a clean new segment descriptor */
1458 set_desc_base(&seg_desc, selector << 4);
1459 set_desc_limit(&seg_desc, 0xffff);
1469 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1470 if ((seg == VCPU_SREG_CS
1471 || (seg == VCPU_SREG_SS
1472 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1473 || seg == VCPU_SREG_TR)
1477 /* TR should be in GDT only */
1478 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1481 if (null_selector) /* for NULL selector skip all following checks */
1484 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1485 if (ret != X86EMUL_CONTINUE)
1488 err_code = selector & 0xfffc;
1489 err_vec = in_task_switch ? TS_VECTOR : GP_VECTOR;
1491 /* can't load system descriptor into segment selector */
1492 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1496 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1505 * segment is not a writable data segment or segment
1506 * selector's RPL != CPL or segment selector's RPL != CPL
1508 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1512 if (!(seg_desc.type & 8))
1515 if (seg_desc.type & 4) {
1521 if (rpl > cpl || dpl != cpl)
1524 /* in long-mode d/b must be clear if l is set */
1525 if (seg_desc.d && seg_desc.l) {
1528 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1529 if (efer & EFER_LMA)
1533 /* CS(RPL) <- CPL */
1534 selector = (selector & 0xfffc) | cpl;
1537 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1539 old_desc = seg_desc;
1540 seg_desc.type |= 2; /* busy */
1541 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1542 sizeof(seg_desc), &ctxt->exception);
1543 if (ret != X86EMUL_CONTINUE)
1546 case VCPU_SREG_LDTR:
1547 if (seg_desc.s || seg_desc.type != 2)
1550 default: /* DS, ES, FS, or GS */
1552 * segment is not a data or readable code segment or
1553 * ((segment is a data or nonconforming code segment)
1554 * and (both RPL and CPL > DPL))
1556 if ((seg_desc.type & 0xa) == 0x8 ||
1557 (((seg_desc.type & 0xc) != 0xc) &&
1558 (rpl > dpl && cpl > dpl)))
1564 /* mark segment as accessed */
1566 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1567 if (ret != X86EMUL_CONTINUE)
1569 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1570 ret = ctxt->ops->read_std(ctxt, desc_addr+8, &base3,
1571 sizeof(base3), &ctxt->exception);
1572 if (ret != X86EMUL_CONTINUE)
1576 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1577 return X86EMUL_CONTINUE;
1579 return emulate_exception(ctxt, err_vec, err_code, true);
1582 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1583 u16 selector, int seg)
1585 u8 cpl = ctxt->ops->cpl(ctxt);
1586 return __load_segment_descriptor(ctxt, selector, seg, cpl, false);
1589 static void write_register_operand(struct operand *op)
1591 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1592 switch (op->bytes) {
1594 *(u8 *)op->addr.reg = (u8)op->val;
1597 *(u16 *)op->addr.reg = (u16)op->val;
1600 *op->addr.reg = (u32)op->val;
1601 break; /* 64b: zero-extend */
1603 *op->addr.reg = op->val;
1608 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1612 write_register_operand(op);
1615 if (ctxt->lock_prefix)
1616 return segmented_cmpxchg(ctxt,
1622 return segmented_write(ctxt,
1628 return segmented_write(ctxt,
1631 op->bytes * op->count);
1634 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1637 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1645 return X86EMUL_CONTINUE;
1648 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1650 struct segmented_address addr;
1652 rsp_increment(ctxt, -bytes);
1653 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1654 addr.seg = VCPU_SREG_SS;
1656 return segmented_write(ctxt, addr, data, bytes);
1659 static int em_push(struct x86_emulate_ctxt *ctxt)
1661 /* Disable writeback. */
1662 ctxt->dst.type = OP_NONE;
1663 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1666 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1667 void *dest, int len)
1670 struct segmented_address addr;
1672 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1673 addr.seg = VCPU_SREG_SS;
1674 rc = segmented_read(ctxt, addr, dest, len);
1675 if (rc != X86EMUL_CONTINUE)
1678 rsp_increment(ctxt, len);
1682 static int em_pop(struct x86_emulate_ctxt *ctxt)
1684 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1687 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1688 void *dest, int len)
1691 unsigned long val, change_mask;
1692 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1693 int cpl = ctxt->ops->cpl(ctxt);
1695 rc = emulate_pop(ctxt, &val, len);
1696 if (rc != X86EMUL_CONTINUE)
1699 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1700 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_AC | EFLG_ID;
1702 switch(ctxt->mode) {
1703 case X86EMUL_MODE_PROT64:
1704 case X86EMUL_MODE_PROT32:
1705 case X86EMUL_MODE_PROT16:
1707 change_mask |= EFLG_IOPL;
1709 change_mask |= EFLG_IF;
1711 case X86EMUL_MODE_VM86:
1713 return emulate_gp(ctxt, 0);
1714 change_mask |= EFLG_IF;
1716 default: /* real mode */
1717 change_mask |= (EFLG_IOPL | EFLG_IF);
1721 *(unsigned long *)dest =
1722 (ctxt->eflags & ~change_mask) | (val & change_mask);
1727 static int em_popf(struct x86_emulate_ctxt *ctxt)
1729 ctxt->dst.type = OP_REG;
1730 ctxt->dst.addr.reg = &ctxt->eflags;
1731 ctxt->dst.bytes = ctxt->op_bytes;
1732 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1735 static int em_enter(struct x86_emulate_ctxt *ctxt)
1738 unsigned frame_size = ctxt->src.val;
1739 unsigned nesting_level = ctxt->src2.val & 31;
1743 return X86EMUL_UNHANDLEABLE;
1745 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1746 rc = push(ctxt, &rbp, stack_size(ctxt));
1747 if (rc != X86EMUL_CONTINUE)
1749 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1751 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1752 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1754 return X86EMUL_CONTINUE;
1757 static int em_leave(struct x86_emulate_ctxt *ctxt)
1759 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1761 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1764 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1766 int seg = ctxt->src2.val;
1768 ctxt->src.val = get_segment_selector(ctxt, seg);
1770 return em_push(ctxt);
1773 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1775 int seg = ctxt->src2.val;
1776 unsigned long selector;
1779 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1780 if (rc != X86EMUL_CONTINUE)
1783 if (ctxt->modrm_reg == VCPU_SREG_SS)
1784 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1786 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1790 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1792 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1793 int rc = X86EMUL_CONTINUE;
1794 int reg = VCPU_REGS_RAX;
1796 while (reg <= VCPU_REGS_RDI) {
1797 (reg == VCPU_REGS_RSP) ?
1798 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1801 if (rc != X86EMUL_CONTINUE)
1810 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1812 ctxt->src.val = (unsigned long)ctxt->eflags;
1813 return em_push(ctxt);
1816 static int em_popa(struct x86_emulate_ctxt *ctxt)
1818 int rc = X86EMUL_CONTINUE;
1819 int reg = VCPU_REGS_RDI;
1821 while (reg >= VCPU_REGS_RAX) {
1822 if (reg == VCPU_REGS_RSP) {
1823 rsp_increment(ctxt, ctxt->op_bytes);
1827 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1828 if (rc != X86EMUL_CONTINUE)
1835 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1837 const struct x86_emulate_ops *ops = ctxt->ops;
1844 /* TODO: Add limit checks */
1845 ctxt->src.val = ctxt->eflags;
1847 if (rc != X86EMUL_CONTINUE)
1850 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1852 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1854 if (rc != X86EMUL_CONTINUE)
1857 ctxt->src.val = ctxt->_eip;
1859 if (rc != X86EMUL_CONTINUE)
1862 ops->get_idt(ctxt, &dt);
1864 eip_addr = dt.address + (irq << 2);
1865 cs_addr = dt.address + (irq << 2) + 2;
1867 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1868 if (rc != X86EMUL_CONTINUE)
1871 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1872 if (rc != X86EMUL_CONTINUE)
1875 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1876 if (rc != X86EMUL_CONTINUE)
1884 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1888 invalidate_registers(ctxt);
1889 rc = __emulate_int_real(ctxt, irq);
1890 if (rc == X86EMUL_CONTINUE)
1891 writeback_registers(ctxt);
1895 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1897 switch(ctxt->mode) {
1898 case X86EMUL_MODE_REAL:
1899 return __emulate_int_real(ctxt, irq);
1900 case X86EMUL_MODE_VM86:
1901 case X86EMUL_MODE_PROT16:
1902 case X86EMUL_MODE_PROT32:
1903 case X86EMUL_MODE_PROT64:
1905 /* Protected mode interrupts unimplemented yet */
1906 return X86EMUL_UNHANDLEABLE;
1910 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1912 int rc = X86EMUL_CONTINUE;
1913 unsigned long temp_eip = 0;
1914 unsigned long temp_eflags = 0;
1915 unsigned long cs = 0;
1916 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1917 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1918 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1919 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1921 /* TODO: Add stack limit check */
1923 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1925 if (rc != X86EMUL_CONTINUE)
1928 if (temp_eip & ~0xffff)
1929 return emulate_gp(ctxt, 0);
1931 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1933 if (rc != X86EMUL_CONTINUE)
1936 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1938 if (rc != X86EMUL_CONTINUE)
1941 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1943 if (rc != X86EMUL_CONTINUE)
1946 ctxt->_eip = temp_eip;
1949 if (ctxt->op_bytes == 4)
1950 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1951 else if (ctxt->op_bytes == 2) {
1952 ctxt->eflags &= ~0xffff;
1953 ctxt->eflags |= temp_eflags;
1956 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1957 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1962 static int em_iret(struct x86_emulate_ctxt *ctxt)
1964 switch(ctxt->mode) {
1965 case X86EMUL_MODE_REAL:
1966 return emulate_iret_real(ctxt);
1967 case X86EMUL_MODE_VM86:
1968 case X86EMUL_MODE_PROT16:
1969 case X86EMUL_MODE_PROT32:
1970 case X86EMUL_MODE_PROT64:
1972 /* iret from protected mode unimplemented yet */
1973 return X86EMUL_UNHANDLEABLE;
1977 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1982 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1984 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1985 if (rc != X86EMUL_CONTINUE)
1989 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1990 return X86EMUL_CONTINUE;
1993 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1995 int rc = X86EMUL_CONTINUE;
1997 switch (ctxt->modrm_reg) {
1998 case 2: /* call near abs */ {
2000 old_eip = ctxt->_eip;
2001 ctxt->_eip = ctxt->src.val;
2002 ctxt->src.val = old_eip;
2006 case 4: /* jmp abs */
2007 ctxt->_eip = ctxt->src.val;
2009 case 5: /* jmp far */
2010 rc = em_jmp_far(ctxt);
2019 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2021 u64 old = ctxt->dst.orig_val64;
2023 if (ctxt->dst.bytes == 16)
2024 return X86EMUL_UNHANDLEABLE;
2026 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2027 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2028 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2029 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2030 ctxt->eflags &= ~EFLG_ZF;
2032 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2033 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2035 ctxt->eflags |= EFLG_ZF;
2037 return X86EMUL_CONTINUE;
2040 static int em_ret(struct x86_emulate_ctxt *ctxt)
2042 ctxt->dst.type = OP_REG;
2043 ctxt->dst.addr.reg = &ctxt->_eip;
2044 ctxt->dst.bytes = ctxt->op_bytes;
2045 return em_pop(ctxt);
2048 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2052 int cpl = ctxt->ops->cpl(ctxt);
2054 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2055 if (rc != X86EMUL_CONTINUE)
2057 if (ctxt->op_bytes == 4)
2058 ctxt->_eip = (u32)ctxt->_eip;
2059 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2060 if (rc != X86EMUL_CONTINUE)
2062 /* Outer-privilege level return is not implemented */
2063 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2064 return X86EMUL_UNHANDLEABLE;
2065 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2069 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2073 rc = em_ret_far(ctxt);
2074 if (rc != X86EMUL_CONTINUE)
2076 rsp_increment(ctxt, ctxt->src.val);
2077 return X86EMUL_CONTINUE;
2080 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2082 /* Save real source value, then compare EAX against destination. */
2083 ctxt->dst.orig_val = ctxt->dst.val;
2084 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2085 ctxt->src.orig_val = ctxt->src.val;
2086 ctxt->src.val = ctxt->dst.orig_val;
2087 fastop(ctxt, em_cmp);
2089 if (ctxt->eflags & EFLG_ZF) {
2090 /* Success: write back to memory. */
2091 ctxt->dst.val = ctxt->src.orig_val;
2093 /* Failure: write the value we saw to EAX. */
2094 ctxt->dst.type = OP_REG;
2095 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2096 ctxt->dst.val = ctxt->dst.orig_val;
2098 return X86EMUL_CONTINUE;
2101 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2103 int seg = ctxt->src2.val;
2107 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2109 rc = load_segment_descriptor(ctxt, sel, seg);
2110 if (rc != X86EMUL_CONTINUE)
2113 ctxt->dst.val = ctxt->src.val;
2118 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2119 struct desc_struct *cs, struct desc_struct *ss)
2121 cs->l = 0; /* will be adjusted later */
2122 set_desc_base(cs, 0); /* flat segment */
2123 cs->g = 1; /* 4kb granularity */
2124 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2125 cs->type = 0x0b; /* Read, Execute, Accessed */
2127 cs->dpl = 0; /* will be adjusted later */
2132 set_desc_base(ss, 0); /* flat segment */
2133 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2134 ss->g = 1; /* 4kb granularity */
2136 ss->type = 0x03; /* Read/Write, Accessed */
2137 ss->d = 1; /* 32bit stack segment */
2144 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2146 u32 eax, ebx, ecx, edx;
2149 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2150 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2151 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2152 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2155 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2157 const struct x86_emulate_ops *ops = ctxt->ops;
2158 u32 eax, ebx, ecx, edx;
2161 * syscall should always be enabled in longmode - so only become
2162 * vendor specific (cpuid) if other modes are active...
2164 if (ctxt->mode == X86EMUL_MODE_PROT64)
2169 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2171 * Intel ("GenuineIntel")
2172 * remark: Intel CPUs only support "syscall" in 64bit
2173 * longmode. Also an 64bit guest with a
2174 * 32bit compat-app running will #UD !! While this
2175 * behaviour can be fixed (by emulating) into AMD
2176 * response - CPUs of AMD can't behave like Intel.
2178 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2179 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2180 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2183 /* AMD ("AuthenticAMD") */
2184 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2185 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2186 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2189 /* AMD ("AMDisbetter!") */
2190 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2191 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2192 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2195 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2199 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2201 const struct x86_emulate_ops *ops = ctxt->ops;
2202 struct desc_struct cs, ss;
2207 /* syscall is not available in real mode */
2208 if (ctxt->mode == X86EMUL_MODE_REAL ||
2209 ctxt->mode == X86EMUL_MODE_VM86)
2210 return emulate_ud(ctxt);
2212 if (!(em_syscall_is_enabled(ctxt)))
2213 return emulate_ud(ctxt);
2215 ops->get_msr(ctxt, MSR_EFER, &efer);
2216 setup_syscalls_segments(ctxt, &cs, &ss);
2218 if (!(efer & EFER_SCE))
2219 return emulate_ud(ctxt);
2221 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2223 cs_sel = (u16)(msr_data & 0xfffc);
2224 ss_sel = (u16)(msr_data + 8);
2226 if (efer & EFER_LMA) {
2230 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2231 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2233 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2234 if (efer & EFER_LMA) {
2235 #ifdef CONFIG_X86_64
2236 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2239 ctxt->mode == X86EMUL_MODE_PROT64 ?
2240 MSR_LSTAR : MSR_CSTAR, &msr_data);
2241 ctxt->_eip = msr_data;
2243 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2244 ctxt->eflags &= ~msr_data;
2248 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2249 ctxt->_eip = (u32)msr_data;
2251 ctxt->eflags &= ~(EFLG_VM | EFLG_IF);
2254 return X86EMUL_CONTINUE;
2257 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2259 const struct x86_emulate_ops *ops = ctxt->ops;
2260 struct desc_struct cs, ss;
2265 ops->get_msr(ctxt, MSR_EFER, &efer);
2266 /* inject #GP if in real mode */
2267 if (ctxt->mode == X86EMUL_MODE_REAL)
2268 return emulate_gp(ctxt, 0);
2271 * Not recognized on AMD in compat mode (but is recognized in legacy
2274 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2275 && !vendor_intel(ctxt))
2276 return emulate_ud(ctxt);
2278 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2279 * Therefore, we inject an #UD.
2281 if (ctxt->mode == X86EMUL_MODE_PROT64)
2282 return emulate_ud(ctxt);
2284 setup_syscalls_segments(ctxt, &cs, &ss);
2286 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2287 switch (ctxt->mode) {
2288 case X86EMUL_MODE_PROT32:
2289 if ((msr_data & 0xfffc) == 0x0)
2290 return emulate_gp(ctxt, 0);
2292 case X86EMUL_MODE_PROT64:
2293 if (msr_data == 0x0)
2294 return emulate_gp(ctxt, 0);
2300 ctxt->eflags &= ~(EFLG_VM | EFLG_IF);
2301 cs_sel = (u16)msr_data;
2302 cs_sel &= ~SELECTOR_RPL_MASK;
2303 ss_sel = cs_sel + 8;
2304 ss_sel &= ~SELECTOR_RPL_MASK;
2305 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2310 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2311 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2313 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2314 ctxt->_eip = msr_data;
2316 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2317 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2319 return X86EMUL_CONTINUE;
2322 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2324 const struct x86_emulate_ops *ops = ctxt->ops;
2325 struct desc_struct cs, ss;
2328 u16 cs_sel = 0, ss_sel = 0;
2330 /* inject #GP if in real mode or Virtual 8086 mode */
2331 if (ctxt->mode == X86EMUL_MODE_REAL ||
2332 ctxt->mode == X86EMUL_MODE_VM86)
2333 return emulate_gp(ctxt, 0);
2335 setup_syscalls_segments(ctxt, &cs, &ss);
2337 if ((ctxt->rex_prefix & 0x8) != 0x0)
2338 usermode = X86EMUL_MODE_PROT64;
2340 usermode = X86EMUL_MODE_PROT32;
2344 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2346 case X86EMUL_MODE_PROT32:
2347 cs_sel = (u16)(msr_data + 16);
2348 if ((msr_data & 0xfffc) == 0x0)
2349 return emulate_gp(ctxt, 0);
2350 ss_sel = (u16)(msr_data + 24);
2352 case X86EMUL_MODE_PROT64:
2353 cs_sel = (u16)(msr_data + 32);
2354 if (msr_data == 0x0)
2355 return emulate_gp(ctxt, 0);
2356 ss_sel = cs_sel + 8;
2361 cs_sel |= SELECTOR_RPL_MASK;
2362 ss_sel |= SELECTOR_RPL_MASK;
2364 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2365 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2367 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2368 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2370 return X86EMUL_CONTINUE;
2373 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2376 if (ctxt->mode == X86EMUL_MODE_REAL)
2378 if (ctxt->mode == X86EMUL_MODE_VM86)
2380 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2381 return ctxt->ops->cpl(ctxt) > iopl;
2384 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2387 const struct x86_emulate_ops *ops = ctxt->ops;
2388 struct desc_struct tr_seg;
2391 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2392 unsigned mask = (1 << len) - 1;
2395 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2398 if (desc_limit_scaled(&tr_seg) < 103)
2400 base = get_desc_base(&tr_seg);
2401 #ifdef CONFIG_X86_64
2402 base |= ((u64)base3) << 32;
2404 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2405 if (r != X86EMUL_CONTINUE)
2407 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2409 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2410 if (r != X86EMUL_CONTINUE)
2412 if ((perm >> bit_idx) & mask)
2417 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2423 if (emulator_bad_iopl(ctxt))
2424 if (!emulator_io_port_access_allowed(ctxt, port, len))
2427 ctxt->perm_ok = true;
2432 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2433 struct tss_segment_16 *tss)
2435 tss->ip = ctxt->_eip;
2436 tss->flag = ctxt->eflags;
2437 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2438 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2439 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2440 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2441 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2442 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2443 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2444 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2446 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2447 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2448 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2449 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2450 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2453 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2454 struct tss_segment_16 *tss)
2459 ctxt->_eip = tss->ip;
2460 ctxt->eflags = tss->flag | 2;
2461 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2462 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2463 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2464 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2465 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2466 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2467 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2468 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2471 * SDM says that segment selectors are loaded before segment
2474 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2475 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2476 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2477 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2478 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2483 * Now load segment descriptors. If fault happens at this stage
2484 * it is handled in a context of new task
2486 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl, true);
2487 if (ret != X86EMUL_CONTINUE)
2489 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, true);
2490 if (ret != X86EMUL_CONTINUE)
2492 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, true);
2493 if (ret != X86EMUL_CONTINUE)
2495 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, true);
2496 if (ret != X86EMUL_CONTINUE)
2498 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, true);
2499 if (ret != X86EMUL_CONTINUE)
2502 return X86EMUL_CONTINUE;
2505 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2506 u16 tss_selector, u16 old_tss_sel,
2507 ulong old_tss_base, struct desc_struct *new_desc)
2509 const struct x86_emulate_ops *ops = ctxt->ops;
2510 struct tss_segment_16 tss_seg;
2512 u32 new_tss_base = get_desc_base(new_desc);
2514 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2516 if (ret != X86EMUL_CONTINUE)
2517 /* FIXME: need to provide precise fault address */
2520 save_state_to_tss16(ctxt, &tss_seg);
2522 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2524 if (ret != X86EMUL_CONTINUE)
2525 /* FIXME: need to provide precise fault address */
2528 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2530 if (ret != X86EMUL_CONTINUE)
2531 /* FIXME: need to provide precise fault address */
2534 if (old_tss_sel != 0xffff) {
2535 tss_seg.prev_task_link = old_tss_sel;
2537 ret = ops->write_std(ctxt, new_tss_base,
2538 &tss_seg.prev_task_link,
2539 sizeof tss_seg.prev_task_link,
2541 if (ret != X86EMUL_CONTINUE)
2542 /* FIXME: need to provide precise fault address */
2546 return load_state_from_tss16(ctxt, &tss_seg);
2549 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2550 struct tss_segment_32 *tss)
2552 /* CR3 and ldt selector are not saved intentionally */
2553 tss->eip = ctxt->_eip;
2554 tss->eflags = ctxt->eflags;
2555 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2556 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2557 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2558 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2559 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2560 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2561 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2562 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2564 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2565 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2566 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2567 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2568 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2569 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2572 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2573 struct tss_segment_32 *tss)
2578 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2579 return emulate_gp(ctxt, 0);
2580 ctxt->_eip = tss->eip;
2581 ctxt->eflags = tss->eflags | 2;
2583 /* General purpose registers */
2584 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2585 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2586 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2587 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2588 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2589 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2590 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2591 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2594 * SDM says that segment selectors are loaded before segment
2595 * descriptors. This is important because CPL checks will
2598 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2599 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2600 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2601 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2602 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2603 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2604 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2607 * If we're switching between Protected Mode and VM86, we need to make
2608 * sure to update the mode before loading the segment descriptors so
2609 * that the selectors are interpreted correctly.
2611 if (ctxt->eflags & X86_EFLAGS_VM) {
2612 ctxt->mode = X86EMUL_MODE_VM86;
2615 ctxt->mode = X86EMUL_MODE_PROT32;
2620 * Now load segment descriptors. If fault happenes at this stage
2621 * it is handled in a context of new task
2623 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR, cpl, true);
2624 if (ret != X86EMUL_CONTINUE)
2626 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, true);
2627 if (ret != X86EMUL_CONTINUE)
2629 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, true);
2630 if (ret != X86EMUL_CONTINUE)
2632 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, true);
2633 if (ret != X86EMUL_CONTINUE)
2635 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, true);
2636 if (ret != X86EMUL_CONTINUE)
2638 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl, true);
2639 if (ret != X86EMUL_CONTINUE)
2641 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl, true);
2642 if (ret != X86EMUL_CONTINUE)
2645 return X86EMUL_CONTINUE;
2648 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2649 u16 tss_selector, u16 old_tss_sel,
2650 ulong old_tss_base, struct desc_struct *new_desc)
2652 const struct x86_emulate_ops *ops = ctxt->ops;
2653 struct tss_segment_32 tss_seg;
2655 u32 new_tss_base = get_desc_base(new_desc);
2656 u32 eip_offset = offsetof(struct tss_segment_32, eip);
2657 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
2659 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2661 if (ret != X86EMUL_CONTINUE)
2662 /* FIXME: need to provide precise fault address */
2665 save_state_to_tss32(ctxt, &tss_seg);
2667 /* Only GP registers and segment selectors are saved */
2668 ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
2669 ldt_sel_offset - eip_offset, &ctxt->exception);
2670 if (ret != X86EMUL_CONTINUE)
2671 /* FIXME: need to provide precise fault address */
2674 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2676 if (ret != X86EMUL_CONTINUE)
2677 /* FIXME: need to provide precise fault address */
2680 if (old_tss_sel != 0xffff) {
2681 tss_seg.prev_task_link = old_tss_sel;
2683 ret = ops->write_std(ctxt, new_tss_base,
2684 &tss_seg.prev_task_link,
2685 sizeof tss_seg.prev_task_link,
2687 if (ret != X86EMUL_CONTINUE)
2688 /* FIXME: need to provide precise fault address */
2692 return load_state_from_tss32(ctxt, &tss_seg);
2695 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2696 u16 tss_selector, int idt_index, int reason,
2697 bool has_error_code, u32 error_code)
2699 const struct x86_emulate_ops *ops = ctxt->ops;
2700 struct desc_struct curr_tss_desc, next_tss_desc;
2702 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2703 ulong old_tss_base =
2704 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2708 /* FIXME: old_tss_base == ~0 ? */
2710 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2711 if (ret != X86EMUL_CONTINUE)
2713 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2714 if (ret != X86EMUL_CONTINUE)
2717 /* FIXME: check that next_tss_desc is tss */
2720 * Check privileges. The three cases are task switch caused by...
2722 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2723 * 2. Exception/IRQ/iret: No check is performed
2724 * 3. jmp/call to TSS: Check against DPL of the TSS
2726 if (reason == TASK_SWITCH_GATE) {
2727 if (idt_index != -1) {
2728 /* Software interrupts */
2729 struct desc_struct task_gate_desc;
2732 ret = read_interrupt_descriptor(ctxt, idt_index,
2734 if (ret != X86EMUL_CONTINUE)
2737 dpl = task_gate_desc.dpl;
2738 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2739 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2741 } else if (reason != TASK_SWITCH_IRET) {
2742 int dpl = next_tss_desc.dpl;
2743 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2744 return emulate_gp(ctxt, tss_selector);
2748 desc_limit = desc_limit_scaled(&next_tss_desc);
2749 if (!next_tss_desc.p ||
2750 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2751 desc_limit < 0x2b)) {
2752 return emulate_ts(ctxt, tss_selector & 0xfffc);
2755 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2756 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2757 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2760 if (reason == TASK_SWITCH_IRET)
2761 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2763 /* set back link to prev task only if NT bit is set in eflags
2764 note that old_tss_sel is not used after this point */
2765 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2766 old_tss_sel = 0xffff;
2768 if (next_tss_desc.type & 8)
2769 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2770 old_tss_base, &next_tss_desc);
2772 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2773 old_tss_base, &next_tss_desc);
2774 if (ret != X86EMUL_CONTINUE)
2777 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2778 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2780 if (reason != TASK_SWITCH_IRET) {
2781 next_tss_desc.type |= (1 << 1); /* set busy flag */
2782 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2785 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2786 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2788 if (has_error_code) {
2789 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2790 ctxt->lock_prefix = 0;
2791 ctxt->src.val = (unsigned long) error_code;
2792 ret = em_push(ctxt);
2798 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2799 u16 tss_selector, int idt_index, int reason,
2800 bool has_error_code, u32 error_code)
2804 invalidate_registers(ctxt);
2805 ctxt->_eip = ctxt->eip;
2806 ctxt->dst.type = OP_NONE;
2808 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2809 has_error_code, error_code);
2811 if (rc == X86EMUL_CONTINUE) {
2812 ctxt->eip = ctxt->_eip;
2813 writeback_registers(ctxt);
2816 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2819 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2822 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2824 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2825 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2828 static int em_das(struct x86_emulate_ctxt *ctxt)
2831 bool af, cf, old_cf;
2833 cf = ctxt->eflags & X86_EFLAGS_CF;
2839 af = ctxt->eflags & X86_EFLAGS_AF;
2840 if ((al & 0x0f) > 9 || af) {
2842 cf = old_cf | (al >= 250);
2847 if (old_al > 0x99 || old_cf) {
2853 /* Set PF, ZF, SF */
2854 ctxt->src.type = OP_IMM;
2856 ctxt->src.bytes = 1;
2857 fastop(ctxt, em_or);
2858 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2860 ctxt->eflags |= X86_EFLAGS_CF;
2862 ctxt->eflags |= X86_EFLAGS_AF;
2863 return X86EMUL_CONTINUE;
2866 static int em_aam(struct x86_emulate_ctxt *ctxt)
2870 if (ctxt->src.val == 0)
2871 return emulate_de(ctxt);
2873 al = ctxt->dst.val & 0xff;
2874 ah = al / ctxt->src.val;
2875 al %= ctxt->src.val;
2877 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
2879 /* Set PF, ZF, SF */
2880 ctxt->src.type = OP_IMM;
2882 ctxt->src.bytes = 1;
2883 fastop(ctxt, em_or);
2885 return X86EMUL_CONTINUE;
2888 static int em_aad(struct x86_emulate_ctxt *ctxt)
2890 u8 al = ctxt->dst.val & 0xff;
2891 u8 ah = (ctxt->dst.val >> 8) & 0xff;
2893 al = (al + (ah * ctxt->src.val)) & 0xff;
2895 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
2897 /* Set PF, ZF, SF */
2898 ctxt->src.type = OP_IMM;
2900 ctxt->src.bytes = 1;
2901 fastop(ctxt, em_or);
2903 return X86EMUL_CONTINUE;
2906 static int em_call(struct x86_emulate_ctxt *ctxt)
2908 long rel = ctxt->src.val;
2910 ctxt->src.val = (unsigned long)ctxt->_eip;
2912 return em_push(ctxt);
2915 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2921 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2922 old_eip = ctxt->_eip;
2924 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2925 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2926 return X86EMUL_CONTINUE;
2929 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2931 ctxt->src.val = old_cs;
2933 if (rc != X86EMUL_CONTINUE)
2936 ctxt->src.val = old_eip;
2937 return em_push(ctxt);
2940 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2944 ctxt->dst.type = OP_REG;
2945 ctxt->dst.addr.reg = &ctxt->_eip;
2946 ctxt->dst.bytes = ctxt->op_bytes;
2947 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2948 if (rc != X86EMUL_CONTINUE)
2950 rsp_increment(ctxt, ctxt->src.val);
2951 return X86EMUL_CONTINUE;
2954 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2956 /* Write back the register source. */
2957 ctxt->src.val = ctxt->dst.val;
2958 write_register_operand(&ctxt->src);
2960 /* Write back the memory destination with implicit LOCK prefix. */
2961 ctxt->dst.val = ctxt->src.orig_val;
2962 ctxt->lock_prefix = 1;
2963 return X86EMUL_CONTINUE;
2966 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2968 ctxt->dst.val = ctxt->src2.val;
2969 return fastop(ctxt, em_imul);
2972 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2974 ctxt->dst.type = OP_REG;
2975 ctxt->dst.bytes = ctxt->src.bytes;
2976 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
2977 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2979 return X86EMUL_CONTINUE;
2982 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2986 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2987 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
2988 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
2989 return X86EMUL_CONTINUE;
2992 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
2996 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
2997 return emulate_gp(ctxt, 0);
2998 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
2999 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3000 return X86EMUL_CONTINUE;
3003 static int em_mov(struct x86_emulate_ctxt *ctxt)
3005 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3006 return X86EMUL_CONTINUE;
3009 #define FFL(x) bit(X86_FEATURE_##x)
3011 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3013 u32 ebx, ecx, edx, eax = 1;
3017 * Check MOVBE is set in the guest-visible CPUID leaf.
3019 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3020 if (!(ecx & FFL(MOVBE)))
3021 return emulate_ud(ctxt);
3023 switch (ctxt->op_bytes) {
3026 * From MOVBE definition: "...When the operand size is 16 bits,
3027 * the upper word of the destination register remains unchanged
3030 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3031 * rules so we have to do the operation almost per hand.
3033 tmp = (u16)ctxt->src.val;
3034 ctxt->dst.val &= ~0xffffUL;
3035 ctxt->dst.val |= (unsigned long)swab16(tmp);
3038 ctxt->dst.val = swab32((u32)ctxt->src.val);
3041 ctxt->dst.val = swab64(ctxt->src.val);
3046 return X86EMUL_CONTINUE;
3049 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3051 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3052 return emulate_gp(ctxt, 0);
3054 /* Disable writeback. */
3055 ctxt->dst.type = OP_NONE;
3056 return X86EMUL_CONTINUE;
3059 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3063 if (ctxt->mode == X86EMUL_MODE_PROT64)
3064 val = ctxt->src.val & ~0ULL;
3066 val = ctxt->src.val & ~0U;
3068 /* #UD condition is already handled. */
3069 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3070 return emulate_gp(ctxt, 0);
3072 /* Disable writeback. */
3073 ctxt->dst.type = OP_NONE;
3074 return X86EMUL_CONTINUE;
3077 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3081 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3082 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3083 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3084 return emulate_gp(ctxt, 0);
3086 return X86EMUL_CONTINUE;
3089 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3093 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3094 return emulate_gp(ctxt, 0);
3096 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3097 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3098 return X86EMUL_CONTINUE;
3101 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3103 if (ctxt->modrm_reg > VCPU_SREG_GS)
3104 return emulate_ud(ctxt);
3106 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3107 return X86EMUL_CONTINUE;
3110 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3112 u16 sel = ctxt->src.val;
3114 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3115 return emulate_ud(ctxt);
3117 if (ctxt->modrm_reg == VCPU_SREG_SS)
3118 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3120 /* Disable writeback. */
3121 ctxt->dst.type = OP_NONE;
3122 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3125 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3127 u16 sel = ctxt->src.val;
3129 /* Disable writeback. */
3130 ctxt->dst.type = OP_NONE;
3131 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3134 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3136 u16 sel = ctxt->src.val;
3138 /* Disable writeback. */
3139 ctxt->dst.type = OP_NONE;
3140 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3143 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3148 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3149 if (rc == X86EMUL_CONTINUE)
3150 ctxt->ops->invlpg(ctxt, linear);
3151 /* Disable writeback. */
3152 ctxt->dst.type = OP_NONE;
3153 return X86EMUL_CONTINUE;
3156 static int em_clts(struct x86_emulate_ctxt *ctxt)
3160 cr0 = ctxt->ops->get_cr(ctxt, 0);
3162 ctxt->ops->set_cr(ctxt, 0, cr0);
3163 return X86EMUL_CONTINUE;
3166 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3168 int rc = ctxt->ops->fix_hypercall(ctxt);
3170 if (rc != X86EMUL_CONTINUE)
3173 /* Let the processor re-execute the fixed hypercall */
3174 ctxt->_eip = ctxt->eip;
3175 /* Disable writeback. */
3176 ctxt->dst.type = OP_NONE;
3177 return X86EMUL_CONTINUE;
3180 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3181 void (*get)(struct x86_emulate_ctxt *ctxt,
3182 struct desc_ptr *ptr))
3184 struct desc_ptr desc_ptr;
3186 if (ctxt->mode == X86EMUL_MODE_PROT64)
3188 get(ctxt, &desc_ptr);
3189 if (ctxt->op_bytes == 2) {
3191 desc_ptr.address &= 0x00ffffff;
3193 /* Disable writeback. */
3194 ctxt->dst.type = OP_NONE;
3195 return segmented_write(ctxt, ctxt->dst.addr.mem,
3196 &desc_ptr, 2 + ctxt->op_bytes);
3199 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3201 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3204 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3206 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3209 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3211 struct desc_ptr desc_ptr;
3214 if (ctxt->mode == X86EMUL_MODE_PROT64)
3216 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3217 &desc_ptr.size, &desc_ptr.address,
3219 if (rc != X86EMUL_CONTINUE)
3221 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3222 /* Disable writeback. */
3223 ctxt->dst.type = OP_NONE;
3224 return X86EMUL_CONTINUE;
3227 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3231 rc = ctxt->ops->fix_hypercall(ctxt);
3233 /* Disable writeback. */
3234 ctxt->dst.type = OP_NONE;
3238 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3240 struct desc_ptr desc_ptr;
3243 if (ctxt->mode == X86EMUL_MODE_PROT64)
3245 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3246 &desc_ptr.size, &desc_ptr.address,
3248 if (rc != X86EMUL_CONTINUE)
3250 ctxt->ops->set_idt(ctxt, &desc_ptr);
3251 /* Disable writeback. */
3252 ctxt->dst.type = OP_NONE;
3253 return X86EMUL_CONTINUE;
3256 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3258 if (ctxt->dst.type == OP_MEM)
3259 ctxt->dst.bytes = 2;
3260 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3261 return X86EMUL_CONTINUE;
3264 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3266 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3267 | (ctxt->src.val & 0x0f));
3268 ctxt->dst.type = OP_NONE;
3269 return X86EMUL_CONTINUE;
3272 static int em_loop(struct x86_emulate_ctxt *ctxt)
3274 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3275 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3276 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3277 jmp_rel(ctxt, ctxt->src.val);
3279 return X86EMUL_CONTINUE;
3282 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3284 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3285 jmp_rel(ctxt, ctxt->src.val);
3287 return X86EMUL_CONTINUE;
3290 static int em_in(struct x86_emulate_ctxt *ctxt)
3292 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3294 return X86EMUL_IO_NEEDED;
3296 return X86EMUL_CONTINUE;
3299 static int em_out(struct x86_emulate_ctxt *ctxt)
3301 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3303 /* Disable writeback. */
3304 ctxt->dst.type = OP_NONE;
3305 return X86EMUL_CONTINUE;
3308 static int em_cli(struct x86_emulate_ctxt *ctxt)
3310 if (emulator_bad_iopl(ctxt))
3311 return emulate_gp(ctxt, 0);
3313 ctxt->eflags &= ~X86_EFLAGS_IF;
3314 return X86EMUL_CONTINUE;
3317 static int em_sti(struct x86_emulate_ctxt *ctxt)
3319 if (emulator_bad_iopl(ctxt))
3320 return emulate_gp(ctxt, 0);
3322 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3323 ctxt->eflags |= X86_EFLAGS_IF;
3324 return X86EMUL_CONTINUE;
3327 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3329 u32 eax, ebx, ecx, edx;
3331 eax = reg_read(ctxt, VCPU_REGS_RAX);
3332 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3333 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3334 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3335 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3336 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3337 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3338 return X86EMUL_CONTINUE;
3341 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3345 flags = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF;
3346 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3348 ctxt->eflags &= ~0xffUL;
3349 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3350 return X86EMUL_CONTINUE;
3353 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3355 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3356 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3357 return X86EMUL_CONTINUE;
3360 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3362 switch (ctxt->op_bytes) {
3363 #ifdef CONFIG_X86_64
3365 asm("bswap %0" : "+r"(ctxt->dst.val));
3369 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3372 return X86EMUL_CONTINUE;
3375 static bool valid_cr(int nr)
3387 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3389 if (!valid_cr(ctxt->modrm_reg))
3390 return emulate_ud(ctxt);
3392 return X86EMUL_CONTINUE;
3395 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3397 u64 new_val = ctxt->src.val64;
3398 int cr = ctxt->modrm_reg;
3401 static u64 cr_reserved_bits[] = {
3402 0xffffffff00000000ULL,
3403 0, 0, 0, /* CR3 checked later */
3410 return emulate_ud(ctxt);
3412 if (new_val & cr_reserved_bits[cr])
3413 return emulate_gp(ctxt, 0);
3418 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3419 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3420 return emulate_gp(ctxt, 0);
3422 cr4 = ctxt->ops->get_cr(ctxt, 4);
3423 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3425 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3426 !(cr4 & X86_CR4_PAE))
3427 return emulate_gp(ctxt, 0);
3434 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3435 if (efer & EFER_LMA)
3436 rsvd = CR3_L_MODE_RESERVED_BITS;
3439 return emulate_gp(ctxt, 0);
3444 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3446 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3447 return emulate_gp(ctxt, 0);
3453 return X86EMUL_CONTINUE;
3456 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3460 ctxt->ops->get_dr(ctxt, 7, &dr7);
3462 /* Check if DR7.Global_Enable is set */
3463 return dr7 & (1 << 13);
3466 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3468 int dr = ctxt->modrm_reg;
3472 return emulate_ud(ctxt);
3474 cr4 = ctxt->ops->get_cr(ctxt, 4);
3475 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3476 return emulate_ud(ctxt);
3478 if (check_dr7_gd(ctxt))
3479 return emulate_db(ctxt);
3481 return X86EMUL_CONTINUE;
3484 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3486 u64 new_val = ctxt->src.val64;
3487 int dr = ctxt->modrm_reg;
3489 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3490 return emulate_gp(ctxt, 0);
3492 return check_dr_read(ctxt);
3495 static int check_svme(struct x86_emulate_ctxt *ctxt)
3499 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3501 if (!(efer & EFER_SVME))
3502 return emulate_ud(ctxt);
3504 return X86EMUL_CONTINUE;
3507 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3509 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3511 /* Valid physical address? */
3512 if (rax & 0xffff000000000000ULL)
3513 return emulate_gp(ctxt, 0);
3515 return check_svme(ctxt);
3518 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3520 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3522 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3523 return emulate_ud(ctxt);
3525 return X86EMUL_CONTINUE;
3528 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3530 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3531 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3533 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3534 ctxt->ops->check_pmc(ctxt, rcx))
3535 return emulate_gp(ctxt, 0);
3537 return X86EMUL_CONTINUE;
3540 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3542 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3543 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3544 return emulate_gp(ctxt, 0);
3546 return X86EMUL_CONTINUE;
3549 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3551 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3552 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3553 return emulate_gp(ctxt, 0);
3555 return X86EMUL_CONTINUE;
3558 #define D(_y) { .flags = (_y) }
3559 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
3560 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
3561 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3562 #define N D(NotImpl)
3563 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3564 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3565 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3566 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3567 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3568 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3569 #define II(_f, _e, _i) \
3570 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
3571 #define IIP(_f, _e, _i, _p) \
3572 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
3573 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3574 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3576 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3577 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3578 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3579 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3580 #define I2bvIP(_f, _e, _i, _p) \
3581 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3583 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3584 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3585 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3587 static const struct opcode group7_rm0[] = {
3589 I(SrcNone | Priv | EmulateOnUD, em_vmcall),
3593 static const struct opcode group7_rm1[] = {
3594 DI(SrcNone | Priv, monitor),
3595 DI(SrcNone | Priv, mwait),
3599 static const struct opcode group7_rm3[] = {
3600 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3601 II(SrcNone | Prot | EmulateOnUD, em_vmmcall, vmmcall),
3602 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3603 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3604 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3605 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3606 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3607 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3610 static const struct opcode group7_rm7[] = {
3612 DIP(SrcNone, rdtscp, check_rdtsc),
3616 static const struct opcode group1[] = {
3618 F(Lock | PageTable, em_or),
3621 F(Lock | PageTable, em_and),
3627 static const struct opcode group1A[] = {
3628 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3631 static const struct opcode group2[] = {
3632 F(DstMem | ModRM, em_rol),
3633 F(DstMem | ModRM, em_ror),
3634 F(DstMem | ModRM, em_rcl),
3635 F(DstMem | ModRM, em_rcr),
3636 F(DstMem | ModRM, em_shl),
3637 F(DstMem | ModRM, em_shr),
3638 F(DstMem | ModRM, em_shl),
3639 F(DstMem | ModRM, em_sar),
3642 static const struct opcode group3[] = {
3643 F(DstMem | SrcImm | NoWrite, em_test),
3644 F(DstMem | SrcImm | NoWrite, em_test),
3645 F(DstMem | SrcNone | Lock, em_not),
3646 F(DstMem | SrcNone | Lock, em_neg),
3647 F(DstXacc | Src2Mem, em_mul_ex),
3648 F(DstXacc | Src2Mem, em_imul_ex),
3649 F(DstXacc | Src2Mem, em_div_ex),
3650 F(DstXacc | Src2Mem, em_idiv_ex),
3653 static const struct opcode group4[] = {
3654 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3655 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3659 static const struct opcode group5[] = {
3660 F(DstMem | SrcNone | Lock, em_inc),
3661 F(DstMem | SrcNone | Lock, em_dec),
3662 I(SrcMem | Stack, em_grp45),
3663 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3664 I(SrcMem | Stack, em_grp45),
3665 I(SrcMemFAddr | ImplicitOps, em_grp45),
3666 I(SrcMem | Stack, em_grp45), D(Undefined),
3669 static const struct opcode group6[] = {
3672 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3673 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3677 static const struct group_dual group7 = { {
3678 II(Mov | DstMem, em_sgdt, sgdt),
3679 II(Mov | DstMem, em_sidt, sidt),
3680 II(SrcMem | Priv, em_lgdt, lgdt),
3681 II(SrcMem | Priv, em_lidt, lidt),
3682 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3683 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3684 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3688 N, EXT(0, group7_rm3),
3689 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3690 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3694 static const struct opcode group8[] = {
3696 F(DstMem | SrcImmByte | NoWrite, em_bt),
3697 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3698 F(DstMem | SrcImmByte | Lock, em_btr),
3699 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3702 static const struct group_dual group9 = { {
3703 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3705 N, N, N, N, N, N, N, N,
3708 static const struct opcode group11[] = {
3709 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3713 static const struct gprefix pfx_0f_6f_0f_7f = {
3714 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3717 static const struct gprefix pfx_0f_2b = {
3718 I(0, em_mov), I(0, em_mov), N, N,
3721 static const struct gprefix pfx_0f_28_0f_29 = {
3722 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
3725 static const struct gprefix pfx_0f_e7 = {
3726 N, I(Sse, em_mov), N, N,
3729 static const struct escape escape_d9 = { {
3730 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3733 N, N, N, N, N, N, N, N,
3735 N, N, N, N, N, N, N, N,
3737 N, N, N, N, N, N, N, N,
3739 N, N, N, N, N, N, N, N,
3741 N, N, N, N, N, N, N, N,
3743 N, N, N, N, N, N, N, N,
3745 N, N, N, N, N, N, N, N,
3747 N, N, N, N, N, N, N, N,
3750 static const struct escape escape_db = { {
3751 N, N, N, N, N, N, N, N,
3754 N, N, N, N, N, N, N, N,
3756 N, N, N, N, N, N, N, N,
3758 N, N, N, N, N, N, N, N,
3760 N, N, N, N, N, N, N, N,
3762 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3764 N, N, N, N, N, N, N, N,
3766 N, N, N, N, N, N, N, N,
3768 N, N, N, N, N, N, N, N,
3771 static const struct escape escape_dd = { {
3772 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3775 N, N, N, N, N, N, N, N,
3777 N, N, N, N, N, N, N, N,
3779 N, N, N, N, N, N, N, N,
3781 N, N, N, N, N, N, N, N,
3783 N, N, N, N, N, N, N, N,
3785 N, N, N, N, N, N, N, N,
3787 N, N, N, N, N, N, N, N,
3789 N, N, N, N, N, N, N, N,
3792 static const struct opcode opcode_table[256] = {
3794 F6ALU(Lock, em_add),
3795 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3796 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3798 F6ALU(Lock | PageTable, em_or),
3799 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3802 F6ALU(Lock, em_adc),
3803 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3804 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3806 F6ALU(Lock, em_sbb),
3807 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3808 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3810 F6ALU(Lock | PageTable, em_and), N, N,
3812 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3814 F6ALU(Lock, em_xor), N, N,
3816 F6ALU(NoWrite, em_cmp), N, N,
3818 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
3820 X8(I(SrcReg | Stack, em_push)),
3822 X8(I(DstReg | Stack, em_pop)),
3824 I(ImplicitOps | Stack | No64, em_pusha),
3825 I(ImplicitOps | Stack | No64, em_popa),
3826 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3829 I(SrcImm | Mov | Stack, em_push),
3830 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3831 I(SrcImmByte | Mov | Stack, em_push),
3832 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3833 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3834 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3838 G(ByteOp | DstMem | SrcImm, group1),
3839 G(DstMem | SrcImm, group1),
3840 G(ByteOp | DstMem | SrcImm | No64, group1),
3841 G(DstMem | SrcImmByte, group1),
3842 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3843 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3845 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3846 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3847 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3848 D(ModRM | SrcMem | NoAccess | DstReg),
3849 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3852 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3854 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3855 I(SrcImmFAddr | No64, em_call_far), N,
3856 II(ImplicitOps | Stack, em_pushf, pushf),
3857 II(ImplicitOps | Stack, em_popf, popf),
3858 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
3860 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3861 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3862 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3863 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3865 F2bv(DstAcc | SrcImm | NoWrite, em_test),
3866 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3867 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3868 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3870 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3872 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3874 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
3875 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3876 I(ImplicitOps | Stack, em_ret),
3877 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3878 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3879 G(ByteOp, group11), G(0, group11),
3881 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3882 I(ImplicitOps | Stack | SrcImmU16, em_ret_far_imm),
3883 I(ImplicitOps | Stack, em_ret_far),
3884 D(ImplicitOps), DI(SrcImmByte, intn),
3885 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3887 G(Src2One | ByteOp, group2), G(Src2One, group2),
3888 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
3889 I(DstAcc | SrcImmUByte | No64, em_aam),
3890 I(DstAcc | SrcImmUByte | No64, em_aad),
3891 F(DstAcc | ByteOp | No64, em_salc),
3892 I(DstAcc | SrcXLat | ByteOp, em_mov),
3894 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3896 X3(I(SrcImmByte, em_loop)),
3897 I(SrcImmByte, em_jcxz),
3898 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3899 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3901 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3902 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3903 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3904 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3906 N, DI(ImplicitOps, icebp), N, N,
3907 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3908 G(ByteOp, group3), G(0, group3),
3910 D(ImplicitOps), D(ImplicitOps),
3911 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3912 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3915 static const struct opcode twobyte_table[256] = {
3917 G(0, group6), GD(0, &group7), N, N,
3918 N, I(ImplicitOps | EmulateOnUD, em_syscall),
3919 II(ImplicitOps | Priv, em_clts, clts), N,
3920 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3921 N, D(ImplicitOps | ModRM), N, N,
3923 N, N, N, N, N, N, N, N,
3924 D(ImplicitOps | ModRM), N, N, N, N, N, N, D(ImplicitOps | ModRM),
3926 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
3927 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
3928 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
3930 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
3933 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
3934 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
3935 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
3938 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3939 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3940 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3941 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3942 I(ImplicitOps | EmulateOnUD, em_sysenter),
3943 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
3945 N, N, N, N, N, N, N, N,
3947 X16(D(DstReg | SrcMem | ModRM)),
3949 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3954 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3959 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3963 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3965 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3966 II(ImplicitOps, em_cpuid, cpuid),
3967 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
3968 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
3969 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
3971 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3972 DI(ImplicitOps, rsm),
3973 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3974 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
3975 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
3976 D(ModRM), F(DstReg | SrcMem | ModRM, em_imul),
3978 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3979 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3980 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3981 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3982 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3983 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3987 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3988 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
3989 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3991 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
3992 N, D(DstMem | SrcReg | ModRM | Mov),
3993 N, N, N, GD(0, &group9),
3995 X8(I(DstReg, em_bswap)),
3997 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3999 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4000 N, N, N, N, N, N, N, N,
4002 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4005 static const struct gprefix three_byte_0f_38_f0 = {
4006 I(DstReg | SrcMem | Mov, em_movbe), N, N, N
4009 static const struct gprefix three_byte_0f_38_f1 = {
4010 I(DstMem | SrcReg | Mov, em_movbe), N, N, N
4014 * Insns below are selected by the prefix which indexed by the third opcode
4017 static const struct opcode opcode_map_0f_38[256] = {
4019 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4021 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4023 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f0),
4024 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f1),
4043 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4047 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4053 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4054 unsigned size, bool sign_extension)
4056 int rc = X86EMUL_CONTINUE;
4060 op->addr.mem.ea = ctxt->_eip;
4061 /* NB. Immediates are sign-extended as necessary. */
4062 switch (op->bytes) {
4064 op->val = insn_fetch(s8, ctxt);
4067 op->val = insn_fetch(s16, ctxt);
4070 op->val = insn_fetch(s32, ctxt);
4073 op->val = insn_fetch(s64, ctxt);
4076 if (!sign_extension) {
4077 switch (op->bytes) {
4085 op->val &= 0xffffffff;
4093 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4096 int rc = X86EMUL_CONTINUE;
4100 decode_register_operand(ctxt, op);
4103 rc = decode_imm(ctxt, op, 1, false);
4106 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4110 if (ctxt->d & BitOp)
4111 fetch_bit_operand(ctxt);
4112 op->orig_val = op->val;
4115 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4119 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4120 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4121 fetch_register_operand(op);
4122 op->orig_val = op->val;
4126 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4127 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4128 fetch_register_operand(op);
4129 op->orig_val = op->val;
4132 if (ctxt->d & ByteOp) {
4137 op->bytes = ctxt->op_bytes;
4138 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4139 fetch_register_operand(op);
4140 op->orig_val = op->val;
4144 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4146 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4147 op->addr.mem.seg = VCPU_SREG_ES;
4154 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4155 fetch_register_operand(op);
4159 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4162 rc = decode_imm(ctxt, op, 1, true);
4169 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4172 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4175 ctxt->memop.bytes = 1;
4176 if (ctxt->memop.type == OP_REG) {
4177 ctxt->memop.addr.reg = decode_register(ctxt,
4178 ctxt->modrm_rm, true);
4179 fetch_register_operand(&ctxt->memop);
4183 ctxt->memop.bytes = 2;
4186 ctxt->memop.bytes = 4;
4189 rc = decode_imm(ctxt, op, 2, false);
4192 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4196 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4198 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4199 op->addr.mem.seg = ctxt->seg_override;
4205 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4207 register_address(ctxt,
4208 reg_read(ctxt, VCPU_REGS_RBX) +
4209 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4210 op->addr.mem.seg = ctxt->seg_override;
4215 op->addr.mem.ea = ctxt->_eip;
4216 op->bytes = ctxt->op_bytes + 2;
4217 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4220 ctxt->memop.bytes = ctxt->op_bytes + 2;
4223 op->val = VCPU_SREG_ES;
4226 op->val = VCPU_SREG_CS;
4229 op->val = VCPU_SREG_SS;
4232 op->val = VCPU_SREG_DS;
4235 op->val = VCPU_SREG_FS;
4238 op->val = VCPU_SREG_GS;
4241 /* Special instructions do their own operand decoding. */
4243 op->type = OP_NONE; /* Disable writeback. */
4251 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4253 int rc = X86EMUL_CONTINUE;
4254 int mode = ctxt->mode;
4255 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4256 bool op_prefix = false;
4257 bool has_seg_override = false;
4258 struct opcode opcode;
4260 ctxt->memop.type = OP_NONE;
4261 ctxt->memopp = NULL;
4262 ctxt->_eip = ctxt->eip;
4263 ctxt->fetch.ptr = ctxt->fetch.data;
4264 ctxt->fetch.end = ctxt->fetch.data + insn_len;
4265 ctxt->opcode_len = 1;
4267 memcpy(ctxt->fetch.data, insn, insn_len);
4269 rc = __do_insn_fetch_bytes(ctxt, 1);
4270 if (rc != X86EMUL_CONTINUE)
4275 case X86EMUL_MODE_REAL:
4276 case X86EMUL_MODE_VM86:
4277 case X86EMUL_MODE_PROT16:
4278 def_op_bytes = def_ad_bytes = 2;
4280 case X86EMUL_MODE_PROT32:
4281 def_op_bytes = def_ad_bytes = 4;
4283 #ifdef CONFIG_X86_64
4284 case X86EMUL_MODE_PROT64:
4290 return EMULATION_FAILED;
4293 ctxt->op_bytes = def_op_bytes;
4294 ctxt->ad_bytes = def_ad_bytes;
4296 /* Legacy prefixes. */
4298 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4299 case 0x66: /* operand-size override */
4301 /* switch between 2/4 bytes */
4302 ctxt->op_bytes = def_op_bytes ^ 6;
4304 case 0x67: /* address-size override */
4305 if (mode == X86EMUL_MODE_PROT64)
4306 /* switch between 4/8 bytes */
4307 ctxt->ad_bytes = def_ad_bytes ^ 12;
4309 /* switch between 2/4 bytes */
4310 ctxt->ad_bytes = def_ad_bytes ^ 6;
4312 case 0x26: /* ES override */
4313 case 0x2e: /* CS override */
4314 case 0x36: /* SS override */
4315 case 0x3e: /* DS override */
4316 has_seg_override = true;
4317 ctxt->seg_override = (ctxt->b >> 3) & 3;
4319 case 0x64: /* FS override */
4320 case 0x65: /* GS override */
4321 has_seg_override = true;
4322 ctxt->seg_override = ctxt->b & 7;
4324 case 0x40 ... 0x4f: /* REX */
4325 if (mode != X86EMUL_MODE_PROT64)
4327 ctxt->rex_prefix = ctxt->b;
4329 case 0xf0: /* LOCK */
4330 ctxt->lock_prefix = 1;
4332 case 0xf2: /* REPNE/REPNZ */
4333 case 0xf3: /* REP/REPE/REPZ */
4334 ctxt->rep_prefix = ctxt->b;
4340 /* Any legacy prefix after a REX prefix nullifies its effect. */
4342 ctxt->rex_prefix = 0;
4348 if (ctxt->rex_prefix & 8)
4349 ctxt->op_bytes = 8; /* REX.W */
4351 /* Opcode byte(s). */
4352 opcode = opcode_table[ctxt->b];
4353 /* Two-byte opcode? */
4354 if (ctxt->b == 0x0f) {
4355 ctxt->opcode_len = 2;
4356 ctxt->b = insn_fetch(u8, ctxt);
4357 opcode = twobyte_table[ctxt->b];
4359 /* 0F_38 opcode map */
4360 if (ctxt->b == 0x38) {
4361 ctxt->opcode_len = 3;
4362 ctxt->b = insn_fetch(u8, ctxt);
4363 opcode = opcode_map_0f_38[ctxt->b];
4366 ctxt->d = opcode.flags;
4368 if (ctxt->d & ModRM)
4369 ctxt->modrm = insn_fetch(u8, ctxt);
4371 /* vex-prefix instructions are not implemented */
4372 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
4373 (mode == X86EMUL_MODE_PROT64 ||
4374 (mode >= X86EMUL_MODE_PROT16 && (ctxt->modrm & 0x80)))) {
4378 while (ctxt->d & GroupMask) {
4379 switch (ctxt->d & GroupMask) {
4381 goffset = (ctxt->modrm >> 3) & 7;
4382 opcode = opcode.u.group[goffset];
4385 goffset = (ctxt->modrm >> 3) & 7;
4386 if ((ctxt->modrm >> 6) == 3)
4387 opcode = opcode.u.gdual->mod3[goffset];
4389 opcode = opcode.u.gdual->mod012[goffset];
4392 goffset = ctxt->modrm & 7;
4393 opcode = opcode.u.group[goffset];
4396 if (ctxt->rep_prefix && op_prefix)
4397 return EMULATION_FAILED;
4398 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4399 switch (simd_prefix) {
4400 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4401 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4402 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4403 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4407 if (ctxt->modrm > 0xbf)
4408 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4410 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4413 return EMULATION_FAILED;
4416 ctxt->d &= ~(u64)GroupMask;
4417 ctxt->d |= opcode.flags;
4422 return EMULATION_FAILED;
4424 ctxt->execute = opcode.u.execute;
4426 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
4427 return EMULATION_FAILED;
4429 if (unlikely(ctxt->d &
4430 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm))) {
4432 * These are copied unconditionally here, and checked unconditionally
4433 * in x86_emulate_insn.
4435 ctxt->check_perm = opcode.check_perm;
4436 ctxt->intercept = opcode.intercept;
4438 if (ctxt->d & NotImpl)
4439 return EMULATION_FAILED;
4441 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4444 if (ctxt->d & Op3264) {
4445 if (mode == X86EMUL_MODE_PROT64)
4452 ctxt->op_bytes = 16;
4453 else if (ctxt->d & Mmx)
4457 /* ModRM and SIB bytes. */
4458 if (ctxt->d & ModRM) {
4459 rc = decode_modrm(ctxt, &ctxt->memop);
4460 if (!has_seg_override) {
4461 has_seg_override = true;
4462 ctxt->seg_override = ctxt->modrm_seg;
4464 } else if (ctxt->d & MemAbs)
4465 rc = decode_abs(ctxt, &ctxt->memop);
4466 if (rc != X86EMUL_CONTINUE)
4469 if (!has_seg_override)
4470 ctxt->seg_override = VCPU_SREG_DS;
4472 ctxt->memop.addr.mem.seg = ctxt->seg_override;
4475 * Decode and fetch the source operand: register, memory
4478 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4479 if (rc != X86EMUL_CONTINUE)
4483 * Decode and fetch the second source operand: register, memory
4486 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4487 if (rc != X86EMUL_CONTINUE)
4490 /* Decode and fetch the destination operand: register or memory. */
4491 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4494 if (ctxt->rip_relative)
4495 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4497 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4500 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4502 return ctxt->d & PageTable;
4505 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4507 /* The second termination condition only applies for REPE
4508 * and REPNE. Test if the repeat string operation prefix is
4509 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4510 * corresponding termination condition according to:
4511 * - if REPE/REPZ and ZF = 0 then done
4512 * - if REPNE/REPNZ and ZF = 1 then done
4514 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4515 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4516 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4517 ((ctxt->eflags & EFLG_ZF) == 0))
4518 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4519 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4525 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4529 ctxt->ops->get_fpu(ctxt);
4530 asm volatile("1: fwait \n\t"
4532 ".pushsection .fixup,\"ax\" \n\t"
4534 "movb $1, %[fault] \n\t"
4537 _ASM_EXTABLE(1b, 3b)
4538 : [fault]"+qm"(fault));
4539 ctxt->ops->put_fpu(ctxt);
4541 if (unlikely(fault))
4542 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4544 return X86EMUL_CONTINUE;
4547 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4550 if (op->type == OP_MM)
4551 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4554 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4556 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4557 if (!(ctxt->d & ByteOp))
4558 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4559 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4560 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
4562 : "c"(ctxt->src2.val));
4563 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4564 if (!fop) /* exception is returned in fop variable */
4565 return emulate_de(ctxt);
4566 return X86EMUL_CONTINUE;
4569 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
4571 memset(&ctxt->rip_relative, 0,
4572 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
4574 ctxt->io_read.pos = 0;
4575 ctxt->io_read.end = 0;
4576 ctxt->mem_read.end = 0;
4579 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4581 const struct x86_emulate_ops *ops = ctxt->ops;
4582 int rc = X86EMUL_CONTINUE;
4583 int saved_dst_type = ctxt->dst.type;
4585 ctxt->mem_read.pos = 0;
4587 /* LOCK prefix is allowed only with some instructions */
4588 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4589 rc = emulate_ud(ctxt);
4593 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4594 rc = emulate_ud(ctxt);
4598 if (unlikely(ctxt->d &
4599 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
4600 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4601 (ctxt->d & Undefined)) {
4602 rc = emulate_ud(ctxt);
4606 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4607 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4608 rc = emulate_ud(ctxt);
4612 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4613 rc = emulate_nm(ctxt);
4617 if (ctxt->d & Mmx) {
4618 rc = flush_pending_x87_faults(ctxt);
4619 if (rc != X86EMUL_CONTINUE)
4622 * Now that we know the fpu is exception safe, we can fetch
4625 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4626 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4627 if (!(ctxt->d & Mov))
4628 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4631 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4632 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4633 X86_ICPT_PRE_EXCEPT);
4634 if (rc != X86EMUL_CONTINUE)
4638 /* Privileged instruction can be executed only in CPL=0 */
4639 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4640 if (ctxt->d & PrivUD)
4641 rc = emulate_ud(ctxt);
4643 rc = emulate_gp(ctxt, 0);
4647 /* Instruction can only be executed in protected mode */
4648 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4649 rc = emulate_ud(ctxt);
4653 /* Do instruction specific permission checks */
4654 if (ctxt->d & CheckPerm) {
4655 rc = ctxt->check_perm(ctxt);
4656 if (rc != X86EMUL_CONTINUE)
4660 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4661 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4662 X86_ICPT_POST_EXCEPT);
4663 if (rc != X86EMUL_CONTINUE)
4667 if (ctxt->rep_prefix && (ctxt->d & String)) {
4668 /* All REP prefixes have the same first termination condition */
4669 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4670 ctxt->eip = ctxt->_eip;
4671 ctxt->eflags &= ~EFLG_RF;
4677 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4678 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4679 ctxt->src.valptr, ctxt->src.bytes);
4680 if (rc != X86EMUL_CONTINUE)
4682 ctxt->src.orig_val64 = ctxt->src.val64;
4685 if (ctxt->src2.type == OP_MEM) {
4686 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4687 &ctxt->src2.val, ctxt->src2.bytes);
4688 if (rc != X86EMUL_CONTINUE)
4692 if ((ctxt->d & DstMask) == ImplicitOps)
4696 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4697 /* optimisation - avoid slow emulated read if Mov */
4698 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4699 &ctxt->dst.val, ctxt->dst.bytes);
4700 if (rc != X86EMUL_CONTINUE)
4703 ctxt->dst.orig_val = ctxt->dst.val;
4707 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4708 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4709 X86_ICPT_POST_MEMACCESS);
4710 if (rc != X86EMUL_CONTINUE)
4714 if (ctxt->rep_prefix && (ctxt->d & String))
4715 ctxt->eflags |= EFLG_RF;
4717 ctxt->eflags &= ~EFLG_RF;
4719 if (ctxt->execute) {
4720 if (ctxt->d & Fastop) {
4721 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4722 rc = fastop(ctxt, fop);
4723 if (rc != X86EMUL_CONTINUE)
4727 rc = ctxt->execute(ctxt);
4728 if (rc != X86EMUL_CONTINUE)
4733 if (ctxt->opcode_len == 2)
4735 else if (ctxt->opcode_len == 3)
4736 goto threebyte_insn;
4739 case 0x63: /* movsxd */
4740 if (ctxt->mode != X86EMUL_MODE_PROT64)
4741 goto cannot_emulate;
4742 ctxt->dst.val = (s32) ctxt->src.val;
4744 case 0x70 ... 0x7f: /* jcc (short) */
4745 if (test_cc(ctxt->b, ctxt->eflags))
4746 jmp_rel(ctxt, ctxt->src.val);
4748 case 0x8d: /* lea r16/r32, m */
4749 ctxt->dst.val = ctxt->src.addr.mem.ea;
4751 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4752 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4753 ctxt->dst.type = OP_NONE;
4757 case 0x98: /* cbw/cwde/cdqe */
4758 switch (ctxt->op_bytes) {
4759 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4760 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4761 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4764 case 0xcc: /* int3 */
4765 rc = emulate_int(ctxt, 3);
4767 case 0xcd: /* int n */
4768 rc = emulate_int(ctxt, ctxt->src.val);
4770 case 0xce: /* into */
4771 if (ctxt->eflags & EFLG_OF)
4772 rc = emulate_int(ctxt, 4);
4774 case 0xe9: /* jmp rel */
4775 case 0xeb: /* jmp rel short */
4776 jmp_rel(ctxt, ctxt->src.val);
4777 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4779 case 0xf4: /* hlt */
4780 ctxt->ops->halt(ctxt);
4782 case 0xf5: /* cmc */
4783 /* complement carry flag from eflags reg */
4784 ctxt->eflags ^= EFLG_CF;
4786 case 0xf8: /* clc */
4787 ctxt->eflags &= ~EFLG_CF;
4789 case 0xf9: /* stc */
4790 ctxt->eflags |= EFLG_CF;
4792 case 0xfc: /* cld */
4793 ctxt->eflags &= ~EFLG_DF;
4795 case 0xfd: /* std */
4796 ctxt->eflags |= EFLG_DF;
4799 goto cannot_emulate;
4802 if (rc != X86EMUL_CONTINUE)
4806 if (ctxt->d & SrcWrite) {
4807 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
4808 rc = writeback(ctxt, &ctxt->src);
4809 if (rc != X86EMUL_CONTINUE)
4812 if (!(ctxt->d & NoWrite)) {
4813 rc = writeback(ctxt, &ctxt->dst);
4814 if (rc != X86EMUL_CONTINUE)
4819 * restore dst type in case the decoding will be reused
4820 * (happens for string instruction )
4822 ctxt->dst.type = saved_dst_type;
4824 if ((ctxt->d & SrcMask) == SrcSI)
4825 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4827 if ((ctxt->d & DstMask) == DstDI)
4828 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4830 if (ctxt->rep_prefix && (ctxt->d & String)) {
4832 struct read_cache *r = &ctxt->io_read;
4833 if ((ctxt->d & SrcMask) == SrcSI)
4834 count = ctxt->src.count;
4836 count = ctxt->dst.count;
4837 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4840 if (!string_insn_completed(ctxt)) {
4842 * Re-enter guest when pio read ahead buffer is empty
4843 * or, if it is not used, after each 1024 iteration.
4845 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4846 (r->end == 0 || r->end != r->pos)) {
4848 * Reset read cache. Usually happens before
4849 * decode, but since instruction is restarted
4850 * we have to do it here.
4852 ctxt->mem_read.end = 0;
4853 writeback_registers(ctxt);
4854 return EMULATION_RESTART;
4856 goto done; /* skip rip writeback */
4858 ctxt->eflags &= ~EFLG_RF;
4861 ctxt->eip = ctxt->_eip;
4864 if (rc == X86EMUL_PROPAGATE_FAULT) {
4865 WARN_ON(ctxt->exception.vector > 0x1f);
4866 ctxt->have_exception = true;
4868 if (rc == X86EMUL_INTERCEPTED)
4869 return EMULATION_INTERCEPTED;
4871 if (rc == X86EMUL_CONTINUE)
4872 writeback_registers(ctxt);
4874 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4878 case 0x09: /* wbinvd */
4879 (ctxt->ops->wbinvd)(ctxt);
4881 case 0x08: /* invd */
4882 case 0x0d: /* GrpP (prefetch) */
4883 case 0x18: /* Grp16 (prefetch/nop) */
4884 case 0x1f: /* nop */
4886 case 0x20: /* mov cr, reg */
4887 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4889 case 0x21: /* mov from dr to reg */
4890 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4892 case 0x40 ... 0x4f: /* cmov */
4893 if (test_cc(ctxt->b, ctxt->eflags))
4894 ctxt->dst.val = ctxt->src.val;
4895 else if (ctxt->mode != X86EMUL_MODE_PROT64 ||
4896 ctxt->op_bytes != 4)
4897 ctxt->dst.type = OP_NONE; /* no writeback */
4899 case 0x80 ... 0x8f: /* jnz rel, etc*/
4900 if (test_cc(ctxt->b, ctxt->eflags))
4901 jmp_rel(ctxt, ctxt->src.val);
4903 case 0x90 ... 0x9f: /* setcc r/m8 */
4904 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4906 case 0xae: /* clflush */
4908 case 0xb6 ... 0xb7: /* movzx */
4909 ctxt->dst.bytes = ctxt->op_bytes;
4910 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4911 : (u16) ctxt->src.val;
4913 case 0xbe ... 0xbf: /* movsx */
4914 ctxt->dst.bytes = ctxt->op_bytes;
4915 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4916 (s16) ctxt->src.val;
4918 case 0xc3: /* movnti */
4919 ctxt->dst.bytes = ctxt->op_bytes;
4920 ctxt->dst.val = (ctxt->op_bytes == 8) ? (u64) ctxt->src.val :
4921 (u32) ctxt->src.val;
4924 goto cannot_emulate;
4929 if (rc != X86EMUL_CONTINUE)
4935 return EMULATION_FAILED;
4938 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4940 invalidate_registers(ctxt);
4943 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4945 writeback_registers(ctxt);