1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
35 #define OpImplicit 1ull /* No generic decode */
36 #define OpReg 2ull /* Register */
37 #define OpMem 3ull /* Memory */
38 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
39 #define OpDI 5ull /* ES:DI/EDI/RDI */
40 #define OpMem64 6ull /* Memory, 64-bit */
41 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
42 #define OpDX 8ull /* DX register */
43 #define OpCL 9ull /* CL register (for shifts) */
44 #define OpImmByte 10ull /* 8-bit sign extended immediate */
45 #define OpOne 11ull /* Implied 1 */
46 #define OpImm 12ull /* Sign extended immediate */
47 #define OpMem16 13ull /* Memory operand (16-bit). */
48 #define OpMem32 14ull /* Memory operand (32-bit). */
49 #define OpImmU 15ull /* Immediate operand, zero extended */
50 #define OpSI 16ull /* SI/ESI/RSI */
51 #define OpImmFAddr 17ull /* Immediate far address */
52 #define OpMemFAddr 18ull /* Far address in memory */
53 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
54 #define OpES 20ull /* ES */
55 #define OpCS 21ull /* CS */
56 #define OpSS 22ull /* SS */
57 #define OpDS 23ull /* DS */
58 #define OpFS 24ull /* FS */
59 #define OpGS 25ull /* GS */
60 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpBits 5 /* Width of operand field */
63 #define OpMask ((1ull << OpBits) - 1)
66 * Opcode effective-address decode tables.
67 * Note that we only emulate instructions that have at least one memory
68 * operand (excluding implicit stack references). We assume that stack
69 * references and instruction fetches will never occur in special memory
70 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
74 /* Operand sizes: 8-bit operands or specified/overridden size. */
75 #define ByteOp (1<<0) /* 8-bit operands. */
76 /* Destination operand type. */
78 #define ImplicitOps (OpImplicit << DstShift)
79 #define DstReg (OpReg << DstShift)
80 #define DstMem (OpMem << DstShift)
81 #define DstAcc (OpAcc << DstShift)
82 #define DstDI (OpDI << DstShift)
83 #define DstMem64 (OpMem64 << DstShift)
84 #define DstImmUByte (OpImmUByte << DstShift)
85 #define DstDX (OpDX << DstShift)
86 #define DstMask (OpMask << DstShift)
87 /* Source operand type. */
89 #define SrcNone (OpNone << SrcShift)
90 #define SrcReg (OpReg << SrcShift)
91 #define SrcMem (OpMem << SrcShift)
92 #define SrcMem16 (OpMem16 << SrcShift)
93 #define SrcMem32 (OpMem32 << SrcShift)
94 #define SrcImm (OpImm << SrcShift)
95 #define SrcImmByte (OpImmByte << SrcShift)
96 #define SrcOne (OpOne << SrcShift)
97 #define SrcImmUByte (OpImmUByte << SrcShift)
98 #define SrcImmU (OpImmU << SrcShift)
99 #define SrcSI (OpSI << SrcShift)
100 #define SrcImmFAddr (OpImmFAddr << SrcShift)
101 #define SrcMemFAddr (OpMemFAddr << SrcShift)
102 #define SrcAcc (OpAcc << SrcShift)
103 #define SrcImmU16 (OpImmU16 << SrcShift)
104 #define SrcDX (OpDX << SrcShift)
105 #define SrcMem8 (OpMem8 << SrcShift)
106 #define SrcMask (OpMask << SrcShift)
107 #define BitOp (1<<11)
108 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
109 #define String (1<<13) /* String instruction (rep capable) */
110 #define Stack (1<<14) /* Stack instruction (push/pop) */
111 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
112 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
113 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
114 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
115 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
116 #define Sse (1<<18) /* SSE Vector instruction */
117 /* Generic ModRM decode. */
118 #define ModRM (1<<19)
119 /* Destination is only written; never read. */
122 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
123 #define VendorSpecific (1<<22) /* Vendor specific instruction */
124 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
125 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
126 #define Undefined (1<<25) /* No Such Instruction */
127 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
128 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
130 #define PageTable (1 << 29) /* instruction used to write page table */
131 /* Source 2 operand type */
132 #define Src2Shift (30)
133 #define Src2None (OpNone << Src2Shift)
134 #define Src2CL (OpCL << Src2Shift)
135 #define Src2ImmByte (OpImmByte << Src2Shift)
136 #define Src2One (OpOne << Src2Shift)
137 #define Src2Imm (OpImm << Src2Shift)
138 #define Src2ES (OpES << Src2Shift)
139 #define Src2CS (OpCS << Src2Shift)
140 #define Src2SS (OpSS << Src2Shift)
141 #define Src2DS (OpDS << Src2Shift)
142 #define Src2FS (OpFS << Src2Shift)
143 #define Src2GS (OpGS << Src2Shift)
144 #define Src2Mask (OpMask << Src2Shift)
145 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
146 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
147 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
148 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
150 #define X2(x...) x, x
151 #define X3(x...) X2(x), x
152 #define X4(x...) X2(x), X2(x)
153 #define X5(x...) X4(x), x
154 #define X6(x...) X4(x), X2(x)
155 #define X7(x...) X4(x), X3(x)
156 #define X8(x...) X4(x), X4(x)
157 #define X16(x...) X8(x), X8(x)
163 int (*execute)(struct x86_emulate_ctxt *ctxt);
164 struct opcode *group;
165 struct group_dual *gdual;
166 struct gprefix *gprefix;
168 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
172 struct opcode mod012[8];
173 struct opcode mod3[8];
177 struct opcode pfx_no;
178 struct opcode pfx_66;
179 struct opcode pfx_f2;
180 struct opcode pfx_f3;
183 /* EFLAGS bit definitions. */
184 #define EFLG_ID (1<<21)
185 #define EFLG_VIP (1<<20)
186 #define EFLG_VIF (1<<19)
187 #define EFLG_AC (1<<18)
188 #define EFLG_VM (1<<17)
189 #define EFLG_RF (1<<16)
190 #define EFLG_IOPL (3<<12)
191 #define EFLG_NT (1<<14)
192 #define EFLG_OF (1<<11)
193 #define EFLG_DF (1<<10)
194 #define EFLG_IF (1<<9)
195 #define EFLG_TF (1<<8)
196 #define EFLG_SF (1<<7)
197 #define EFLG_ZF (1<<6)
198 #define EFLG_AF (1<<4)
199 #define EFLG_PF (1<<2)
200 #define EFLG_CF (1<<0)
202 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
203 #define EFLG_RESERVED_ONE_MASK 2
206 * Instruction emulation:
207 * Most instructions are emulated directly via a fragment of inline assembly
208 * code. This allows us to save/restore EFLAGS and thus very easily pick up
209 * any modified flags.
212 #if defined(CONFIG_X86_64)
213 #define _LO32 "k" /* force 32-bit operand */
214 #define _STK "%%rsp" /* stack pointer */
215 #elif defined(__i386__)
216 #define _LO32 "" /* force 32-bit operand */
217 #define _STK "%%esp" /* stack pointer */
221 * These EFLAGS bits are restored from saved value during emulation, and
222 * any changes are written back to the saved value after emulation.
224 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
226 /* Before executing instruction: restore necessary bits in EFLAGS. */
227 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
228 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
229 "movl %"_sav",%"_LO32 _tmp"; " \
232 "movl %"_msk",%"_LO32 _tmp"; " \
233 "andl %"_LO32 _tmp",("_STK"); " \
235 "notl %"_LO32 _tmp"; " \
236 "andl %"_LO32 _tmp",("_STK"); " \
237 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
239 "orl %"_LO32 _tmp",("_STK"); " \
243 /* After executing instruction: write-back necessary bits in EFLAGS. */
244 #define _POST_EFLAGS(_sav, _msk, _tmp) \
245 /* _sav |= EFLAGS & _msk; */ \
248 "andl %"_msk",%"_LO32 _tmp"; " \
249 "orl %"_LO32 _tmp",%"_sav"; "
257 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
259 __asm__ __volatile__ ( \
260 _PRE_EFLAGS("0", "4", "2") \
261 _op _suffix " %"_x"3,%1; " \
262 _POST_EFLAGS("0", "4", "2") \
263 : "=m" ((ctxt)->eflags), \
264 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
266 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
270 /* Raw emulation: instruction has two explicit operands. */
271 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
273 unsigned long _tmp; \
275 switch ((ctxt)->dst.bytes) { \
277 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
280 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
283 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
288 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
290 unsigned long _tmp; \
291 switch ((ctxt)->dst.bytes) { \
293 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
296 __emulate_2op_nobyte(ctxt, _op, \
297 _wx, _wy, _lx, _ly, _qx, _qy); \
302 /* Source operand is byte-sized and may be restricted to just %cl. */
303 #define emulate_2op_SrcB(ctxt, _op) \
304 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
306 /* Source operand is byte, word, long or quad sized. */
307 #define emulate_2op_SrcV(ctxt, _op) \
308 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
310 /* Source operand is word, long or quad sized. */
311 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
312 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
314 /* Instruction has three operands and one operand is stored in ECX register */
315 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
317 unsigned long _tmp; \
318 _type _clv = (ctxt)->src2.val; \
319 _type _srcv = (ctxt)->src.val; \
320 _type _dstv = (ctxt)->dst.val; \
322 __asm__ __volatile__ ( \
323 _PRE_EFLAGS("0", "5", "2") \
324 _op _suffix " %4,%1 \n" \
325 _POST_EFLAGS("0", "5", "2") \
326 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
327 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
330 (ctxt)->src2.val = (unsigned long) _clv; \
331 (ctxt)->src2.val = (unsigned long) _srcv; \
332 (ctxt)->dst.val = (unsigned long) _dstv; \
335 #define emulate_2op_cl(ctxt, _op) \
337 switch ((ctxt)->dst.bytes) { \
339 __emulate_2op_cl(ctxt, _op, "w", u16); \
342 __emulate_2op_cl(ctxt, _op, "l", u32); \
345 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
350 #define __emulate_1op(ctxt, _op, _suffix) \
352 unsigned long _tmp; \
354 __asm__ __volatile__ ( \
355 _PRE_EFLAGS("0", "3", "2") \
356 _op _suffix " %1; " \
357 _POST_EFLAGS("0", "3", "2") \
358 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
360 : "i" (EFLAGS_MASK)); \
363 /* Instruction has only one explicit operand (no source operand). */
364 #define emulate_1op(ctxt, _op) \
366 switch ((ctxt)->dst.bytes) { \
367 case 1: __emulate_1op(ctxt, _op, "b"); break; \
368 case 2: __emulate_1op(ctxt, _op, "w"); break; \
369 case 4: __emulate_1op(ctxt, _op, "l"); break; \
370 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
374 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
376 unsigned long _tmp; \
377 ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX]; \
378 ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX]; \
380 __asm__ __volatile__ ( \
381 _PRE_EFLAGS("0", "5", "1") \
383 _op _suffix " %6; " \
385 _POST_EFLAGS("0", "5", "1") \
386 ".pushsection .fixup,\"ax\" \n\t" \
387 "3: movb $1, %4 \n\t" \
390 _ASM_EXTABLE(1b, 3b) \
391 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
392 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
393 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val), \
394 "a" (*rax), "d" (*rdx)); \
397 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
398 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
400 switch((ctxt)->src.bytes) { \
402 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
405 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
408 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
411 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
416 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
417 enum x86_intercept intercept,
418 enum x86_intercept_stage stage)
420 struct x86_instruction_info info = {
421 .intercept = intercept,
422 .rep_prefix = ctxt->rep_prefix,
423 .modrm_mod = ctxt->modrm_mod,
424 .modrm_reg = ctxt->modrm_reg,
425 .modrm_rm = ctxt->modrm_rm,
426 .src_val = ctxt->src.val64,
427 .src_bytes = ctxt->src.bytes,
428 .dst_bytes = ctxt->dst.bytes,
429 .ad_bytes = ctxt->ad_bytes,
430 .next_rip = ctxt->eip,
433 return ctxt->ops->intercept(ctxt, &info, stage);
436 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
438 return (1UL << (ctxt->ad_bytes << 3)) - 1;
441 /* Access/update address held in a register, based on addressing mode. */
442 static inline unsigned long
443 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
445 if (ctxt->ad_bytes == sizeof(unsigned long))
448 return reg & ad_mask(ctxt);
451 static inline unsigned long
452 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
454 return address_mask(ctxt, reg);
458 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
460 if (ctxt->ad_bytes == sizeof(unsigned long))
463 *reg = (*reg & ~ad_mask(ctxt)) | ((*reg + inc) & ad_mask(ctxt));
466 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
468 register_address_increment(ctxt, &ctxt->_eip, rel);
471 static u32 desc_limit_scaled(struct desc_struct *desc)
473 u32 limit = get_desc_limit(desc);
475 return desc->g ? (limit << 12) | 0xfff : limit;
478 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
480 ctxt->has_seg_override = true;
481 ctxt->seg_override = seg;
484 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
486 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
489 return ctxt->ops->get_cached_segment_base(ctxt, seg);
492 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
494 if (!ctxt->has_seg_override)
497 return ctxt->seg_override;
500 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
501 u32 error, bool valid)
503 ctxt->exception.vector = vec;
504 ctxt->exception.error_code = error;
505 ctxt->exception.error_code_valid = valid;
506 return X86EMUL_PROPAGATE_FAULT;
509 static int emulate_db(struct x86_emulate_ctxt *ctxt)
511 return emulate_exception(ctxt, DB_VECTOR, 0, false);
514 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
516 return emulate_exception(ctxt, GP_VECTOR, err, true);
519 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
521 return emulate_exception(ctxt, SS_VECTOR, err, true);
524 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
526 return emulate_exception(ctxt, UD_VECTOR, 0, false);
529 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
531 return emulate_exception(ctxt, TS_VECTOR, err, true);
534 static int emulate_de(struct x86_emulate_ctxt *ctxt)
536 return emulate_exception(ctxt, DE_VECTOR, 0, false);
539 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
541 return emulate_exception(ctxt, NM_VECTOR, 0, false);
544 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
547 struct desc_struct desc;
549 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
553 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
558 struct desc_struct desc;
560 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
561 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
565 * x86 defines three classes of vector instructions: explicitly
566 * aligned, explicitly unaligned, and the rest, which change behaviour
567 * depending on whether they're AVX encoded or not.
569 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
570 * subject to the same check.
572 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
574 if (likely(size < 16))
577 if (ctxt->d & Aligned)
579 else if (ctxt->d & Unaligned)
581 else if (ctxt->d & Avx)
587 static int __linearize(struct x86_emulate_ctxt *ctxt,
588 struct segmented_address addr,
589 unsigned size, bool write, bool fetch,
592 struct desc_struct desc;
599 la = seg_base(ctxt, addr.seg) + addr.ea;
600 switch (ctxt->mode) {
601 case X86EMUL_MODE_REAL:
603 case X86EMUL_MODE_PROT64:
604 if (((signed long)la << 16) >> 16 != la)
605 return emulate_gp(ctxt, 0);
608 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
612 /* code segment or read-only data segment */
613 if (((desc.type & 8) || !(desc.type & 2)) && write)
615 /* unreadable code segment */
616 if (!fetch && (desc.type & 8) && !(desc.type & 2))
618 lim = desc_limit_scaled(&desc);
619 if ((desc.type & 8) || !(desc.type & 4)) {
620 /* expand-up segment */
621 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
624 /* exapand-down segment */
625 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
627 lim = desc.d ? 0xffffffff : 0xffff;
628 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
631 cpl = ctxt->ops->cpl(ctxt);
634 if (!(desc.type & 8)) {
638 } else if ((desc.type & 8) && !(desc.type & 4)) {
639 /* nonconforming code segment */
642 } else if ((desc.type & 8) && (desc.type & 4)) {
643 /* conforming code segment */
649 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
651 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
652 return emulate_gp(ctxt, 0);
654 return X86EMUL_CONTINUE;
656 if (addr.seg == VCPU_SREG_SS)
657 return emulate_ss(ctxt, addr.seg);
659 return emulate_gp(ctxt, addr.seg);
662 static int linearize(struct x86_emulate_ctxt *ctxt,
663 struct segmented_address addr,
664 unsigned size, bool write,
667 return __linearize(ctxt, addr, size, write, false, linear);
671 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
672 struct segmented_address addr,
679 rc = linearize(ctxt, addr, size, false, &linear);
680 if (rc != X86EMUL_CONTINUE)
682 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
686 * Fetch the next byte of the instruction being emulated which is pointed to
687 * by ctxt->_eip, then increment ctxt->_eip.
689 * Also prefetch the remaining bytes of the instruction without crossing page
690 * boundary if they are not in fetch_cache yet.
692 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
694 struct fetch_cache *fc = &ctxt->fetch;
698 if (ctxt->_eip == fc->end) {
699 unsigned long linear;
700 struct segmented_address addr = { .seg = VCPU_SREG_CS,
702 cur_size = fc->end - fc->start;
703 size = min(15UL - cur_size,
704 PAGE_SIZE - offset_in_page(ctxt->_eip));
705 rc = __linearize(ctxt, addr, size, false, true, &linear);
706 if (unlikely(rc != X86EMUL_CONTINUE))
708 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
709 size, &ctxt->exception);
710 if (unlikely(rc != X86EMUL_CONTINUE))
714 *dest = fc->data[ctxt->_eip - fc->start];
716 return X86EMUL_CONTINUE;
719 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
720 void *dest, unsigned size)
724 /* x86 instructions are limited to 15 bytes. */
725 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
726 return X86EMUL_UNHANDLEABLE;
728 rc = do_insn_fetch_byte(ctxt, dest++);
729 if (rc != X86EMUL_CONTINUE)
732 return X86EMUL_CONTINUE;
735 /* Fetch next part of the instruction being emulated. */
736 #define insn_fetch(_type, _ctxt) \
737 ({ unsigned long _x; \
738 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
739 if (rc != X86EMUL_CONTINUE) \
744 #define insn_fetch_arr(_arr, _size, _ctxt) \
745 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
746 if (rc != X86EMUL_CONTINUE) \
751 * Given the 'reg' portion of a ModRM byte, and a register block, return a
752 * pointer into the block that addresses the relevant register.
753 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
755 static void *decode_register(u8 modrm_reg, unsigned long *regs,
760 p = ®s[modrm_reg];
761 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
762 p = (unsigned char *)®s[modrm_reg & 3] + 1;
766 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
767 struct segmented_address addr,
768 u16 *size, unsigned long *address, int op_bytes)
775 rc = segmented_read_std(ctxt, addr, size, 2);
776 if (rc != X86EMUL_CONTINUE)
779 rc = segmented_read_std(ctxt, addr, address, op_bytes);
783 static int test_cc(unsigned int condition, unsigned int flags)
787 switch ((condition & 15) >> 1) {
789 rc |= (flags & EFLG_OF);
791 case 1: /* b/c/nae */
792 rc |= (flags & EFLG_CF);
795 rc |= (flags & EFLG_ZF);
798 rc |= (flags & (EFLG_CF|EFLG_ZF));
801 rc |= (flags & EFLG_SF);
804 rc |= (flags & EFLG_PF);
807 rc |= (flags & EFLG_ZF);
810 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
814 /* Odd condition identifiers (lsb == 1) have inverted sense. */
815 return (!!rc ^ (condition & 1));
818 static void fetch_register_operand(struct operand *op)
822 op->val = *(u8 *)op->addr.reg;
825 op->val = *(u16 *)op->addr.reg;
828 op->val = *(u32 *)op->addr.reg;
831 op->val = *(u64 *)op->addr.reg;
836 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
838 ctxt->ops->get_fpu(ctxt);
840 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
841 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
842 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
843 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
844 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
845 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
846 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
847 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
849 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
850 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
851 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
852 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
853 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
854 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
855 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
856 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
860 ctxt->ops->put_fpu(ctxt);
863 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
866 ctxt->ops->get_fpu(ctxt);
868 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
869 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
870 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
871 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
872 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
873 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
874 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
875 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
877 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
878 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
879 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
880 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
881 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
882 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
883 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
884 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
888 ctxt->ops->put_fpu(ctxt);
891 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
893 ctxt->ops->get_fpu(ctxt);
895 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
896 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
897 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
898 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
899 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
900 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
901 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
902 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
905 ctxt->ops->put_fpu(ctxt);
908 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
910 ctxt->ops->get_fpu(ctxt);
912 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
913 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
914 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
915 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
916 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
917 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
918 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
919 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
922 ctxt->ops->put_fpu(ctxt);
925 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
928 unsigned reg = ctxt->modrm_reg;
929 int highbyte_regs = ctxt->rex_prefix == 0;
931 if (!(ctxt->d & ModRM))
932 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
938 read_sse_reg(ctxt, &op->vec_val, reg);
950 if (ctxt->d & ByteOp) {
951 op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
954 op->addr.reg = decode_register(reg, ctxt->regs, 0);
955 op->bytes = ctxt->op_bytes;
957 fetch_register_operand(op);
958 op->orig_val = op->val;
961 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
965 int index_reg = 0, base_reg = 0, scale;
966 int rc = X86EMUL_CONTINUE;
969 if (ctxt->rex_prefix) {
970 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
971 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
972 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
975 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
976 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
977 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
978 ctxt->modrm_seg = VCPU_SREG_DS;
980 if (ctxt->modrm_mod == 3) {
982 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
983 op->addr.reg = decode_register(ctxt->modrm_rm,
984 ctxt->regs, ctxt->d & ByteOp);
988 op->addr.xmm = ctxt->modrm_rm;
989 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
995 op->addr.xmm = ctxt->modrm_rm & 7;
998 fetch_register_operand(op);
1004 if (ctxt->ad_bytes == 2) {
1005 unsigned bx = ctxt->regs[VCPU_REGS_RBX];
1006 unsigned bp = ctxt->regs[VCPU_REGS_RBP];
1007 unsigned si = ctxt->regs[VCPU_REGS_RSI];
1008 unsigned di = ctxt->regs[VCPU_REGS_RDI];
1010 /* 16-bit ModR/M decode. */
1011 switch (ctxt->modrm_mod) {
1013 if (ctxt->modrm_rm == 6)
1014 modrm_ea += insn_fetch(u16, ctxt);
1017 modrm_ea += insn_fetch(s8, ctxt);
1020 modrm_ea += insn_fetch(u16, ctxt);
1023 switch (ctxt->modrm_rm) {
1025 modrm_ea += bx + si;
1028 modrm_ea += bx + di;
1031 modrm_ea += bp + si;
1034 modrm_ea += bp + di;
1043 if (ctxt->modrm_mod != 0)
1050 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1051 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1052 ctxt->modrm_seg = VCPU_SREG_SS;
1053 modrm_ea = (u16)modrm_ea;
1055 /* 32/64-bit ModR/M decode. */
1056 if ((ctxt->modrm_rm & 7) == 4) {
1057 sib = insn_fetch(u8, ctxt);
1058 index_reg |= (sib >> 3) & 7;
1059 base_reg |= sib & 7;
1062 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1063 modrm_ea += insn_fetch(s32, ctxt);
1065 modrm_ea += ctxt->regs[base_reg];
1067 modrm_ea += ctxt->regs[index_reg] << scale;
1068 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1069 if (ctxt->mode == X86EMUL_MODE_PROT64)
1070 ctxt->rip_relative = 1;
1072 modrm_ea += ctxt->regs[ctxt->modrm_rm];
1073 switch (ctxt->modrm_mod) {
1075 if (ctxt->modrm_rm == 5)
1076 modrm_ea += insn_fetch(s32, ctxt);
1079 modrm_ea += insn_fetch(s8, ctxt);
1082 modrm_ea += insn_fetch(s32, ctxt);
1086 op->addr.mem.ea = modrm_ea;
1091 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1094 int rc = X86EMUL_CONTINUE;
1097 switch (ctxt->ad_bytes) {
1099 op->addr.mem.ea = insn_fetch(u16, ctxt);
1102 op->addr.mem.ea = insn_fetch(u32, ctxt);
1105 op->addr.mem.ea = insn_fetch(u64, ctxt);
1112 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1116 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1117 mask = ~(ctxt->dst.bytes * 8 - 1);
1119 if (ctxt->src.bytes == 2)
1120 sv = (s16)ctxt->src.val & (s16)mask;
1121 else if (ctxt->src.bytes == 4)
1122 sv = (s32)ctxt->src.val & (s32)mask;
1124 ctxt->dst.addr.mem.ea += (sv >> 3);
1127 /* only subword offset */
1128 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1131 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1132 unsigned long addr, void *dest, unsigned size)
1135 struct read_cache *mc = &ctxt->mem_read;
1138 int n = min(size, 8u);
1140 if (mc->pos < mc->end)
1143 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1145 if (rc != X86EMUL_CONTINUE)
1150 memcpy(dest, mc->data + mc->pos, n);
1155 return X86EMUL_CONTINUE;
1158 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1159 struct segmented_address addr,
1166 rc = linearize(ctxt, addr, size, false, &linear);
1167 if (rc != X86EMUL_CONTINUE)
1169 return read_emulated(ctxt, linear, data, size);
1172 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1173 struct segmented_address addr,
1180 rc = linearize(ctxt, addr, size, true, &linear);
1181 if (rc != X86EMUL_CONTINUE)
1183 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1187 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1188 struct segmented_address addr,
1189 const void *orig_data, const void *data,
1195 rc = linearize(ctxt, addr, size, true, &linear);
1196 if (rc != X86EMUL_CONTINUE)
1198 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1199 size, &ctxt->exception);
1202 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1203 unsigned int size, unsigned short port,
1206 struct read_cache *rc = &ctxt->io_read;
1208 if (rc->pos == rc->end) { /* refill pio read ahead */
1209 unsigned int in_page, n;
1210 unsigned int count = ctxt->rep_prefix ?
1211 address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
1212 in_page = (ctxt->eflags & EFLG_DF) ?
1213 offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
1214 PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
1215 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1219 rc->pos = rc->end = 0;
1220 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1225 memcpy(dest, rc->data + rc->pos, size);
1230 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1231 u16 index, struct desc_struct *desc)
1236 ctxt->ops->get_idt(ctxt, &dt);
1238 if (dt.size < index * 8 + 7)
1239 return emulate_gp(ctxt, index << 3 | 0x2);
1241 addr = dt.address + index * 8;
1242 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1246 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1247 u16 selector, struct desc_ptr *dt)
1249 struct x86_emulate_ops *ops = ctxt->ops;
1251 if (selector & 1 << 2) {
1252 struct desc_struct desc;
1255 memset (dt, 0, sizeof *dt);
1256 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1259 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1260 dt->address = get_desc_base(&desc);
1262 ops->get_gdt(ctxt, dt);
1265 /* allowed just for 8 bytes segments */
1266 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1267 u16 selector, struct desc_struct *desc)
1270 u16 index = selector >> 3;
1273 get_descriptor_table_ptr(ctxt, selector, &dt);
1275 if (dt.size < index * 8 + 7)
1276 return emulate_gp(ctxt, selector & 0xfffc);
1278 addr = dt.address + index * 8;
1279 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1283 /* allowed just for 8 bytes segments */
1284 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1285 u16 selector, struct desc_struct *desc)
1288 u16 index = selector >> 3;
1291 get_descriptor_table_ptr(ctxt, selector, &dt);
1293 if (dt.size < index * 8 + 7)
1294 return emulate_gp(ctxt, selector & 0xfffc);
1296 addr = dt.address + index * 8;
1297 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1301 /* Does not support long mode */
1302 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1303 u16 selector, int seg)
1305 struct desc_struct seg_desc;
1307 unsigned err_vec = GP_VECTOR;
1309 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1312 memset(&seg_desc, 0, sizeof seg_desc);
1314 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1315 || ctxt->mode == X86EMUL_MODE_REAL) {
1316 /* set real mode segment descriptor */
1317 set_desc_base(&seg_desc, selector << 4);
1318 set_desc_limit(&seg_desc, 0xffff);
1322 if (ctxt->mode == X86EMUL_MODE_VM86)
1327 /* NULL selector is not valid for TR, CS and SS */
1328 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1332 /* TR should be in GDT only */
1333 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1336 if (null_selector) /* for NULL selector skip all following checks */
1339 ret = read_segment_descriptor(ctxt, selector, &seg_desc);
1340 if (ret != X86EMUL_CONTINUE)
1343 err_code = selector & 0xfffc;
1344 err_vec = GP_VECTOR;
1346 /* can't load system descriptor into segment selecor */
1347 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1351 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1357 cpl = ctxt->ops->cpl(ctxt);
1362 * segment is not a writable data segment or segment
1363 * selector's RPL != CPL or segment selector's RPL != CPL
1365 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1369 if (!(seg_desc.type & 8))
1372 if (seg_desc.type & 4) {
1378 if (rpl > cpl || dpl != cpl)
1381 /* CS(RPL) <- CPL */
1382 selector = (selector & 0xfffc) | cpl;
1385 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1388 case VCPU_SREG_LDTR:
1389 if (seg_desc.s || seg_desc.type != 2)
1392 default: /* DS, ES, FS, or GS */
1394 * segment is not a data or readable code segment or
1395 * ((segment is a data or nonconforming code segment)
1396 * and (both RPL and CPL > DPL))
1398 if ((seg_desc.type & 0xa) == 0x8 ||
1399 (((seg_desc.type & 0xc) != 0xc) &&
1400 (rpl > dpl && cpl > dpl)))
1406 /* mark segment as accessed */
1408 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1409 if (ret != X86EMUL_CONTINUE)
1413 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1414 return X86EMUL_CONTINUE;
1416 emulate_exception(ctxt, err_vec, err_code, true);
1417 return X86EMUL_PROPAGATE_FAULT;
1420 static void write_register_operand(struct operand *op)
1422 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1423 switch (op->bytes) {
1425 *(u8 *)op->addr.reg = (u8)op->val;
1428 *(u16 *)op->addr.reg = (u16)op->val;
1431 *op->addr.reg = (u32)op->val;
1432 break; /* 64b: zero-extend */
1434 *op->addr.reg = op->val;
1439 static int writeback(struct x86_emulate_ctxt *ctxt)
1443 switch (ctxt->dst.type) {
1445 write_register_operand(&ctxt->dst);
1448 if (ctxt->lock_prefix)
1449 rc = segmented_cmpxchg(ctxt,
1451 &ctxt->dst.orig_val,
1455 rc = segmented_write(ctxt,
1459 if (rc != X86EMUL_CONTINUE)
1463 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1466 write_mmx_reg(ctxt, &ctxt->dst.mm_val, ctxt->dst.addr.mm);
1474 return X86EMUL_CONTINUE;
1477 static int em_push(struct x86_emulate_ctxt *ctxt)
1479 struct segmented_address addr;
1481 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], -ctxt->op_bytes);
1482 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1483 addr.seg = VCPU_SREG_SS;
1485 /* Disable writeback. */
1486 ctxt->dst.type = OP_NONE;
1487 return segmented_write(ctxt, addr, &ctxt->src.val, ctxt->op_bytes);
1490 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1491 void *dest, int len)
1494 struct segmented_address addr;
1496 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1497 addr.seg = VCPU_SREG_SS;
1498 rc = segmented_read(ctxt, addr, dest, len);
1499 if (rc != X86EMUL_CONTINUE)
1502 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], len);
1506 static int em_pop(struct x86_emulate_ctxt *ctxt)
1508 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1511 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1512 void *dest, int len)
1515 unsigned long val, change_mask;
1516 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1517 int cpl = ctxt->ops->cpl(ctxt);
1519 rc = emulate_pop(ctxt, &val, len);
1520 if (rc != X86EMUL_CONTINUE)
1523 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1524 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1526 switch(ctxt->mode) {
1527 case X86EMUL_MODE_PROT64:
1528 case X86EMUL_MODE_PROT32:
1529 case X86EMUL_MODE_PROT16:
1531 change_mask |= EFLG_IOPL;
1533 change_mask |= EFLG_IF;
1535 case X86EMUL_MODE_VM86:
1537 return emulate_gp(ctxt, 0);
1538 change_mask |= EFLG_IF;
1540 default: /* real mode */
1541 change_mask |= (EFLG_IOPL | EFLG_IF);
1545 *(unsigned long *)dest =
1546 (ctxt->eflags & ~change_mask) | (val & change_mask);
1551 static int em_popf(struct x86_emulate_ctxt *ctxt)
1553 ctxt->dst.type = OP_REG;
1554 ctxt->dst.addr.reg = &ctxt->eflags;
1555 ctxt->dst.bytes = ctxt->op_bytes;
1556 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1559 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1561 int seg = ctxt->src2.val;
1563 ctxt->src.val = get_segment_selector(ctxt, seg);
1565 return em_push(ctxt);
1568 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1570 int seg = ctxt->src2.val;
1571 unsigned long selector;
1574 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1575 if (rc != X86EMUL_CONTINUE)
1578 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1582 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1584 unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
1585 int rc = X86EMUL_CONTINUE;
1586 int reg = VCPU_REGS_RAX;
1588 while (reg <= VCPU_REGS_RDI) {
1589 (reg == VCPU_REGS_RSP) ?
1590 (ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
1593 if (rc != X86EMUL_CONTINUE)
1602 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1604 ctxt->src.val = (unsigned long)ctxt->eflags;
1605 return em_push(ctxt);
1608 static int em_popa(struct x86_emulate_ctxt *ctxt)
1610 int rc = X86EMUL_CONTINUE;
1611 int reg = VCPU_REGS_RDI;
1613 while (reg >= VCPU_REGS_RAX) {
1614 if (reg == VCPU_REGS_RSP) {
1615 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP],
1620 rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
1621 if (rc != X86EMUL_CONTINUE)
1628 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1630 struct x86_emulate_ops *ops = ctxt->ops;
1637 /* TODO: Add limit checks */
1638 ctxt->src.val = ctxt->eflags;
1640 if (rc != X86EMUL_CONTINUE)
1643 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1645 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1647 if (rc != X86EMUL_CONTINUE)
1650 ctxt->src.val = ctxt->_eip;
1652 if (rc != X86EMUL_CONTINUE)
1655 ops->get_idt(ctxt, &dt);
1657 eip_addr = dt.address + (irq << 2);
1658 cs_addr = dt.address + (irq << 2) + 2;
1660 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1661 if (rc != X86EMUL_CONTINUE)
1664 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1665 if (rc != X86EMUL_CONTINUE)
1668 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1669 if (rc != X86EMUL_CONTINUE)
1677 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1679 switch(ctxt->mode) {
1680 case X86EMUL_MODE_REAL:
1681 return emulate_int_real(ctxt, irq);
1682 case X86EMUL_MODE_VM86:
1683 case X86EMUL_MODE_PROT16:
1684 case X86EMUL_MODE_PROT32:
1685 case X86EMUL_MODE_PROT64:
1687 /* Protected mode interrupts unimplemented yet */
1688 return X86EMUL_UNHANDLEABLE;
1692 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1694 int rc = X86EMUL_CONTINUE;
1695 unsigned long temp_eip = 0;
1696 unsigned long temp_eflags = 0;
1697 unsigned long cs = 0;
1698 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1699 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1700 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1701 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1703 /* TODO: Add stack limit check */
1705 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1707 if (rc != X86EMUL_CONTINUE)
1710 if (temp_eip & ~0xffff)
1711 return emulate_gp(ctxt, 0);
1713 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1715 if (rc != X86EMUL_CONTINUE)
1718 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1720 if (rc != X86EMUL_CONTINUE)
1723 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1725 if (rc != X86EMUL_CONTINUE)
1728 ctxt->_eip = temp_eip;
1731 if (ctxt->op_bytes == 4)
1732 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1733 else if (ctxt->op_bytes == 2) {
1734 ctxt->eflags &= ~0xffff;
1735 ctxt->eflags |= temp_eflags;
1738 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1739 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1744 static int em_iret(struct x86_emulate_ctxt *ctxt)
1746 switch(ctxt->mode) {
1747 case X86EMUL_MODE_REAL:
1748 return emulate_iret_real(ctxt);
1749 case X86EMUL_MODE_VM86:
1750 case X86EMUL_MODE_PROT16:
1751 case X86EMUL_MODE_PROT32:
1752 case X86EMUL_MODE_PROT64:
1754 /* iret from protected mode unimplemented yet */
1755 return X86EMUL_UNHANDLEABLE;
1759 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1764 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1766 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1767 if (rc != X86EMUL_CONTINUE)
1771 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1772 return X86EMUL_CONTINUE;
1775 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1777 switch (ctxt->modrm_reg) {
1779 emulate_2op_SrcB(ctxt, "rol");
1782 emulate_2op_SrcB(ctxt, "ror");
1785 emulate_2op_SrcB(ctxt, "rcl");
1788 emulate_2op_SrcB(ctxt, "rcr");
1790 case 4: /* sal/shl */
1791 case 6: /* sal/shl */
1792 emulate_2op_SrcB(ctxt, "sal");
1795 emulate_2op_SrcB(ctxt, "shr");
1798 emulate_2op_SrcB(ctxt, "sar");
1801 return X86EMUL_CONTINUE;
1804 static int em_not(struct x86_emulate_ctxt *ctxt)
1806 ctxt->dst.val = ~ctxt->dst.val;
1807 return X86EMUL_CONTINUE;
1810 static int em_neg(struct x86_emulate_ctxt *ctxt)
1812 emulate_1op(ctxt, "neg");
1813 return X86EMUL_CONTINUE;
1816 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
1820 emulate_1op_rax_rdx(ctxt, "mul", ex);
1821 return X86EMUL_CONTINUE;
1824 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
1828 emulate_1op_rax_rdx(ctxt, "imul", ex);
1829 return X86EMUL_CONTINUE;
1832 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1836 emulate_1op_rax_rdx(ctxt, "div", de);
1838 return emulate_de(ctxt);
1839 return X86EMUL_CONTINUE;
1842 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
1846 emulate_1op_rax_rdx(ctxt, "idiv", de);
1848 return emulate_de(ctxt);
1849 return X86EMUL_CONTINUE;
1852 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1854 int rc = X86EMUL_CONTINUE;
1856 switch (ctxt->modrm_reg) {
1858 emulate_1op(ctxt, "inc");
1861 emulate_1op(ctxt, "dec");
1863 case 2: /* call near abs */ {
1865 old_eip = ctxt->_eip;
1866 ctxt->_eip = ctxt->src.val;
1867 ctxt->src.val = old_eip;
1871 case 4: /* jmp abs */
1872 ctxt->_eip = ctxt->src.val;
1874 case 5: /* jmp far */
1875 rc = em_jmp_far(ctxt);
1884 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
1886 u64 old = ctxt->dst.orig_val64;
1888 if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
1889 ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
1890 ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1891 ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1892 ctxt->eflags &= ~EFLG_ZF;
1894 ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
1895 (u32) ctxt->regs[VCPU_REGS_RBX];
1897 ctxt->eflags |= EFLG_ZF;
1899 return X86EMUL_CONTINUE;
1902 static int em_ret(struct x86_emulate_ctxt *ctxt)
1904 ctxt->dst.type = OP_REG;
1905 ctxt->dst.addr.reg = &ctxt->_eip;
1906 ctxt->dst.bytes = ctxt->op_bytes;
1907 return em_pop(ctxt);
1910 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
1915 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
1916 if (rc != X86EMUL_CONTINUE)
1918 if (ctxt->op_bytes == 4)
1919 ctxt->_eip = (u32)ctxt->_eip;
1920 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1921 if (rc != X86EMUL_CONTINUE)
1923 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1927 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
1929 /* Save real source value, then compare EAX against destination. */
1930 ctxt->src.orig_val = ctxt->src.val;
1931 ctxt->src.val = ctxt->regs[VCPU_REGS_RAX];
1932 emulate_2op_SrcV(ctxt, "cmp");
1934 if (ctxt->eflags & EFLG_ZF) {
1935 /* Success: write back to memory. */
1936 ctxt->dst.val = ctxt->src.orig_val;
1938 /* Failure: write the value we saw to EAX. */
1939 ctxt->dst.type = OP_REG;
1940 ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX];
1942 return X86EMUL_CONTINUE;
1945 static int em_lseg(struct x86_emulate_ctxt *ctxt)
1947 int seg = ctxt->src2.val;
1951 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1953 rc = load_segment_descriptor(ctxt, sel, seg);
1954 if (rc != X86EMUL_CONTINUE)
1957 ctxt->dst.val = ctxt->src.val;
1962 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1963 struct desc_struct *cs, struct desc_struct *ss)
1967 memset(cs, 0, sizeof(struct desc_struct));
1968 ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1969 memset(ss, 0, sizeof(struct desc_struct));
1971 cs->l = 0; /* will be adjusted later */
1972 set_desc_base(cs, 0); /* flat segment */
1973 cs->g = 1; /* 4kb granularity */
1974 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1975 cs->type = 0x0b; /* Read, Execute, Accessed */
1977 cs->dpl = 0; /* will be adjusted later */
1981 set_desc_base(ss, 0); /* flat segment */
1982 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1983 ss->g = 1; /* 4kb granularity */
1985 ss->type = 0x03; /* Read/Write, Accessed */
1986 ss->d = 1; /* 32bit stack segment */
1991 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
1993 u32 eax, ebx, ecx, edx;
1996 return ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx)
1997 && ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
1998 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
1999 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2002 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2004 struct x86_emulate_ops *ops = ctxt->ops;
2005 u32 eax, ebx, ecx, edx;
2008 * syscall should always be enabled in longmode - so only become
2009 * vendor specific (cpuid) if other modes are active...
2011 if (ctxt->mode == X86EMUL_MODE_PROT64)
2016 if (ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx)) {
2018 * Intel ("GenuineIntel")
2019 * remark: Intel CPUs only support "syscall" in 64bit
2020 * longmode. Also an 64bit guest with a
2021 * 32bit compat-app running will #UD !! While this
2022 * behaviour can be fixed (by emulating) into AMD
2023 * response - CPUs of AMD can't behave like Intel.
2025 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2026 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2027 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2030 /* AMD ("AuthenticAMD") */
2031 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2032 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2033 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2036 /* AMD ("AMDisbetter!") */
2037 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2038 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2039 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2043 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2047 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2049 struct x86_emulate_ops *ops = ctxt->ops;
2050 struct desc_struct cs, ss;
2055 /* syscall is not available in real mode */
2056 if (ctxt->mode == X86EMUL_MODE_REAL ||
2057 ctxt->mode == X86EMUL_MODE_VM86)
2058 return emulate_ud(ctxt);
2060 if (!(em_syscall_is_enabled(ctxt)))
2061 return emulate_ud(ctxt);
2063 ops->get_msr(ctxt, MSR_EFER, &efer);
2064 setup_syscalls_segments(ctxt, &cs, &ss);
2066 if (!(efer & EFER_SCE))
2067 return emulate_ud(ctxt);
2069 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2071 cs_sel = (u16)(msr_data & 0xfffc);
2072 ss_sel = (u16)(msr_data + 8);
2074 if (efer & EFER_LMA) {
2078 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2079 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2081 ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
2082 if (efer & EFER_LMA) {
2083 #ifdef CONFIG_X86_64
2084 ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
2087 ctxt->mode == X86EMUL_MODE_PROT64 ?
2088 MSR_LSTAR : MSR_CSTAR, &msr_data);
2089 ctxt->_eip = msr_data;
2091 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2092 ctxt->eflags &= ~(msr_data | EFLG_RF);
2096 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2097 ctxt->_eip = (u32)msr_data;
2099 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2102 return X86EMUL_CONTINUE;
2105 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2107 struct x86_emulate_ops *ops = ctxt->ops;
2108 struct desc_struct cs, ss;
2113 ops->get_msr(ctxt, MSR_EFER, &efer);
2114 /* inject #GP if in real mode */
2115 if (ctxt->mode == X86EMUL_MODE_REAL)
2116 return emulate_gp(ctxt, 0);
2119 * Not recognized on AMD in compat mode (but is recognized in legacy
2122 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2123 && !vendor_intel(ctxt))
2124 return emulate_ud(ctxt);
2126 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2127 * Therefore, we inject an #UD.
2129 if (ctxt->mode == X86EMUL_MODE_PROT64)
2130 return emulate_ud(ctxt);
2132 setup_syscalls_segments(ctxt, &cs, &ss);
2134 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2135 switch (ctxt->mode) {
2136 case X86EMUL_MODE_PROT32:
2137 if ((msr_data & 0xfffc) == 0x0)
2138 return emulate_gp(ctxt, 0);
2140 case X86EMUL_MODE_PROT64:
2141 if (msr_data == 0x0)
2142 return emulate_gp(ctxt, 0);
2146 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2147 cs_sel = (u16)msr_data;
2148 cs_sel &= ~SELECTOR_RPL_MASK;
2149 ss_sel = cs_sel + 8;
2150 ss_sel &= ~SELECTOR_RPL_MASK;
2151 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2156 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2157 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2159 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2160 ctxt->_eip = msr_data;
2162 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2163 ctxt->regs[VCPU_REGS_RSP] = msr_data;
2165 return X86EMUL_CONTINUE;
2168 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2170 struct x86_emulate_ops *ops = ctxt->ops;
2171 struct desc_struct cs, ss;
2174 u16 cs_sel = 0, ss_sel = 0;
2176 /* inject #GP if in real mode or Virtual 8086 mode */
2177 if (ctxt->mode == X86EMUL_MODE_REAL ||
2178 ctxt->mode == X86EMUL_MODE_VM86)
2179 return emulate_gp(ctxt, 0);
2181 setup_syscalls_segments(ctxt, &cs, &ss);
2183 if ((ctxt->rex_prefix & 0x8) != 0x0)
2184 usermode = X86EMUL_MODE_PROT64;
2186 usermode = X86EMUL_MODE_PROT32;
2190 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2192 case X86EMUL_MODE_PROT32:
2193 cs_sel = (u16)(msr_data + 16);
2194 if ((msr_data & 0xfffc) == 0x0)
2195 return emulate_gp(ctxt, 0);
2196 ss_sel = (u16)(msr_data + 24);
2198 case X86EMUL_MODE_PROT64:
2199 cs_sel = (u16)(msr_data + 32);
2200 if (msr_data == 0x0)
2201 return emulate_gp(ctxt, 0);
2202 ss_sel = cs_sel + 8;
2207 cs_sel |= SELECTOR_RPL_MASK;
2208 ss_sel |= SELECTOR_RPL_MASK;
2210 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2211 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2213 ctxt->_eip = ctxt->regs[VCPU_REGS_RDX];
2214 ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX];
2216 return X86EMUL_CONTINUE;
2219 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2222 if (ctxt->mode == X86EMUL_MODE_REAL)
2224 if (ctxt->mode == X86EMUL_MODE_VM86)
2226 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2227 return ctxt->ops->cpl(ctxt) > iopl;
2230 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2233 struct x86_emulate_ops *ops = ctxt->ops;
2234 struct desc_struct tr_seg;
2237 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2238 unsigned mask = (1 << len) - 1;
2241 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2244 if (desc_limit_scaled(&tr_seg) < 103)
2246 base = get_desc_base(&tr_seg);
2247 #ifdef CONFIG_X86_64
2248 base |= ((u64)base3) << 32;
2250 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2251 if (r != X86EMUL_CONTINUE)
2253 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2255 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2256 if (r != X86EMUL_CONTINUE)
2258 if ((perm >> bit_idx) & mask)
2263 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2269 if (emulator_bad_iopl(ctxt))
2270 if (!emulator_io_port_access_allowed(ctxt, port, len))
2273 ctxt->perm_ok = true;
2278 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2279 struct tss_segment_16 *tss)
2281 tss->ip = ctxt->_eip;
2282 tss->flag = ctxt->eflags;
2283 tss->ax = ctxt->regs[VCPU_REGS_RAX];
2284 tss->cx = ctxt->regs[VCPU_REGS_RCX];
2285 tss->dx = ctxt->regs[VCPU_REGS_RDX];
2286 tss->bx = ctxt->regs[VCPU_REGS_RBX];
2287 tss->sp = ctxt->regs[VCPU_REGS_RSP];
2288 tss->bp = ctxt->regs[VCPU_REGS_RBP];
2289 tss->si = ctxt->regs[VCPU_REGS_RSI];
2290 tss->di = ctxt->regs[VCPU_REGS_RDI];
2292 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2293 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2294 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2295 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2296 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2299 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2300 struct tss_segment_16 *tss)
2304 ctxt->_eip = tss->ip;
2305 ctxt->eflags = tss->flag | 2;
2306 ctxt->regs[VCPU_REGS_RAX] = tss->ax;
2307 ctxt->regs[VCPU_REGS_RCX] = tss->cx;
2308 ctxt->regs[VCPU_REGS_RDX] = tss->dx;
2309 ctxt->regs[VCPU_REGS_RBX] = tss->bx;
2310 ctxt->regs[VCPU_REGS_RSP] = tss->sp;
2311 ctxt->regs[VCPU_REGS_RBP] = tss->bp;
2312 ctxt->regs[VCPU_REGS_RSI] = tss->si;
2313 ctxt->regs[VCPU_REGS_RDI] = tss->di;
2316 * SDM says that segment selectors are loaded before segment
2319 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2320 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2321 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2322 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2323 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2326 * Now load segment descriptors. If fault happenes at this stage
2327 * it is handled in a context of new task
2329 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2330 if (ret != X86EMUL_CONTINUE)
2332 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2333 if (ret != X86EMUL_CONTINUE)
2335 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2336 if (ret != X86EMUL_CONTINUE)
2338 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2339 if (ret != X86EMUL_CONTINUE)
2341 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2342 if (ret != X86EMUL_CONTINUE)
2345 return X86EMUL_CONTINUE;
2348 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2349 u16 tss_selector, u16 old_tss_sel,
2350 ulong old_tss_base, struct desc_struct *new_desc)
2352 struct x86_emulate_ops *ops = ctxt->ops;
2353 struct tss_segment_16 tss_seg;
2355 u32 new_tss_base = get_desc_base(new_desc);
2357 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2359 if (ret != X86EMUL_CONTINUE)
2360 /* FIXME: need to provide precise fault address */
2363 save_state_to_tss16(ctxt, &tss_seg);
2365 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2367 if (ret != X86EMUL_CONTINUE)
2368 /* FIXME: need to provide precise fault address */
2371 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2373 if (ret != X86EMUL_CONTINUE)
2374 /* FIXME: need to provide precise fault address */
2377 if (old_tss_sel != 0xffff) {
2378 tss_seg.prev_task_link = old_tss_sel;
2380 ret = ops->write_std(ctxt, new_tss_base,
2381 &tss_seg.prev_task_link,
2382 sizeof tss_seg.prev_task_link,
2384 if (ret != X86EMUL_CONTINUE)
2385 /* FIXME: need to provide precise fault address */
2389 return load_state_from_tss16(ctxt, &tss_seg);
2392 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2393 struct tss_segment_32 *tss)
2395 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2396 tss->eip = ctxt->_eip;
2397 tss->eflags = ctxt->eflags;
2398 tss->eax = ctxt->regs[VCPU_REGS_RAX];
2399 tss->ecx = ctxt->regs[VCPU_REGS_RCX];
2400 tss->edx = ctxt->regs[VCPU_REGS_RDX];
2401 tss->ebx = ctxt->regs[VCPU_REGS_RBX];
2402 tss->esp = ctxt->regs[VCPU_REGS_RSP];
2403 tss->ebp = ctxt->regs[VCPU_REGS_RBP];
2404 tss->esi = ctxt->regs[VCPU_REGS_RSI];
2405 tss->edi = ctxt->regs[VCPU_REGS_RDI];
2407 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2408 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2409 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2410 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2411 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2412 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2413 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2416 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2417 struct tss_segment_32 *tss)
2421 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2422 return emulate_gp(ctxt, 0);
2423 ctxt->_eip = tss->eip;
2424 ctxt->eflags = tss->eflags | 2;
2426 /* General purpose registers */
2427 ctxt->regs[VCPU_REGS_RAX] = tss->eax;
2428 ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
2429 ctxt->regs[VCPU_REGS_RDX] = tss->edx;
2430 ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
2431 ctxt->regs[VCPU_REGS_RSP] = tss->esp;
2432 ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
2433 ctxt->regs[VCPU_REGS_RSI] = tss->esi;
2434 ctxt->regs[VCPU_REGS_RDI] = tss->edi;
2437 * SDM says that segment selectors are loaded before segment
2440 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2441 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2442 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2443 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2444 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2445 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2446 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2449 * If we're switching between Protected Mode and VM86, we need to make
2450 * sure to update the mode before loading the segment descriptors so
2451 * that the selectors are interpreted correctly.
2453 * Need to get rflags to the vcpu struct immediately because it
2454 * influences the CPL which is checked at least when loading the segment
2455 * descriptors and when pushing an error code to the new kernel stack.
2457 * TODO Introduce a separate ctxt->ops->set_cpl callback
2459 if (ctxt->eflags & X86_EFLAGS_VM)
2460 ctxt->mode = X86EMUL_MODE_VM86;
2462 ctxt->mode = X86EMUL_MODE_PROT32;
2464 ctxt->ops->set_rflags(ctxt, ctxt->eflags);
2467 * Now load segment descriptors. If fault happenes at this stage
2468 * it is handled in a context of new task
2470 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2471 if (ret != X86EMUL_CONTINUE)
2473 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2474 if (ret != X86EMUL_CONTINUE)
2476 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2477 if (ret != X86EMUL_CONTINUE)
2479 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2480 if (ret != X86EMUL_CONTINUE)
2482 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2483 if (ret != X86EMUL_CONTINUE)
2485 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2486 if (ret != X86EMUL_CONTINUE)
2488 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2489 if (ret != X86EMUL_CONTINUE)
2492 return X86EMUL_CONTINUE;
2495 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2496 u16 tss_selector, u16 old_tss_sel,
2497 ulong old_tss_base, struct desc_struct *new_desc)
2499 struct x86_emulate_ops *ops = ctxt->ops;
2500 struct tss_segment_32 tss_seg;
2502 u32 new_tss_base = get_desc_base(new_desc);
2504 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2506 if (ret != X86EMUL_CONTINUE)
2507 /* FIXME: need to provide precise fault address */
2510 save_state_to_tss32(ctxt, &tss_seg);
2512 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2514 if (ret != X86EMUL_CONTINUE)
2515 /* FIXME: need to provide precise fault address */
2518 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2520 if (ret != X86EMUL_CONTINUE)
2521 /* FIXME: need to provide precise fault address */
2524 if (old_tss_sel != 0xffff) {
2525 tss_seg.prev_task_link = old_tss_sel;
2527 ret = ops->write_std(ctxt, new_tss_base,
2528 &tss_seg.prev_task_link,
2529 sizeof tss_seg.prev_task_link,
2531 if (ret != X86EMUL_CONTINUE)
2532 /* FIXME: need to provide precise fault address */
2536 return load_state_from_tss32(ctxt, &tss_seg);
2539 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2540 u16 tss_selector, int idt_index, int reason,
2541 bool has_error_code, u32 error_code)
2543 struct x86_emulate_ops *ops = ctxt->ops;
2544 struct desc_struct curr_tss_desc, next_tss_desc;
2546 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2547 ulong old_tss_base =
2548 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2551 /* FIXME: old_tss_base == ~0 ? */
2553 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2554 if (ret != X86EMUL_CONTINUE)
2556 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2557 if (ret != X86EMUL_CONTINUE)
2560 /* FIXME: check that next_tss_desc is tss */
2563 * Check privileges. The three cases are task switch caused by...
2565 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2566 * 2. Exception/IRQ/iret: No check is performed
2567 * 3. jmp/call to TSS: Check agains DPL of the TSS
2569 if (reason == TASK_SWITCH_GATE) {
2570 if (idt_index != -1) {
2571 /* Software interrupts */
2572 struct desc_struct task_gate_desc;
2575 ret = read_interrupt_descriptor(ctxt, idt_index,
2577 if (ret != X86EMUL_CONTINUE)
2580 dpl = task_gate_desc.dpl;
2581 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2582 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2584 } else if (reason != TASK_SWITCH_IRET) {
2585 int dpl = next_tss_desc.dpl;
2586 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2587 return emulate_gp(ctxt, tss_selector);
2591 desc_limit = desc_limit_scaled(&next_tss_desc);
2592 if (!next_tss_desc.p ||
2593 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2594 desc_limit < 0x2b)) {
2595 emulate_ts(ctxt, tss_selector & 0xfffc);
2596 return X86EMUL_PROPAGATE_FAULT;
2599 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2600 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2601 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2604 if (reason == TASK_SWITCH_IRET)
2605 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2607 /* set back link to prev task only if NT bit is set in eflags
2608 note that old_tss_sel is not used afetr this point */
2609 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2610 old_tss_sel = 0xffff;
2612 if (next_tss_desc.type & 8)
2613 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2614 old_tss_base, &next_tss_desc);
2616 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2617 old_tss_base, &next_tss_desc);
2618 if (ret != X86EMUL_CONTINUE)
2621 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2622 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2624 if (reason != TASK_SWITCH_IRET) {
2625 next_tss_desc.type |= (1 << 1); /* set busy flag */
2626 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2629 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2630 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2632 if (has_error_code) {
2633 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2634 ctxt->lock_prefix = 0;
2635 ctxt->src.val = (unsigned long) error_code;
2636 ret = em_push(ctxt);
2642 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2643 u16 tss_selector, int idt_index, int reason,
2644 bool has_error_code, u32 error_code)
2648 ctxt->_eip = ctxt->eip;
2649 ctxt->dst.type = OP_NONE;
2651 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2652 has_error_code, error_code);
2654 if (rc == X86EMUL_CONTINUE)
2655 ctxt->eip = ctxt->_eip;
2657 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2660 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2661 int reg, struct operand *op)
2663 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2665 register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
2666 op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
2667 op->addr.mem.seg = seg;
2670 static int em_das(struct x86_emulate_ctxt *ctxt)
2673 bool af, cf, old_cf;
2675 cf = ctxt->eflags & X86_EFLAGS_CF;
2681 af = ctxt->eflags & X86_EFLAGS_AF;
2682 if ((al & 0x0f) > 9 || af) {
2684 cf = old_cf | (al >= 250);
2689 if (old_al > 0x99 || old_cf) {
2695 /* Set PF, ZF, SF */
2696 ctxt->src.type = OP_IMM;
2698 ctxt->src.bytes = 1;
2699 emulate_2op_SrcV(ctxt, "or");
2700 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2702 ctxt->eflags |= X86_EFLAGS_CF;
2704 ctxt->eflags |= X86_EFLAGS_AF;
2705 return X86EMUL_CONTINUE;
2708 static int em_call(struct x86_emulate_ctxt *ctxt)
2710 long rel = ctxt->src.val;
2712 ctxt->src.val = (unsigned long)ctxt->_eip;
2714 return em_push(ctxt);
2717 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2723 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2724 old_eip = ctxt->_eip;
2726 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2727 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2728 return X86EMUL_CONTINUE;
2731 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2733 ctxt->src.val = old_cs;
2735 if (rc != X86EMUL_CONTINUE)
2738 ctxt->src.val = old_eip;
2739 return em_push(ctxt);
2742 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2746 ctxt->dst.type = OP_REG;
2747 ctxt->dst.addr.reg = &ctxt->_eip;
2748 ctxt->dst.bytes = ctxt->op_bytes;
2749 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2750 if (rc != X86EMUL_CONTINUE)
2752 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], ctxt->src.val);
2753 return X86EMUL_CONTINUE;
2756 static int em_add(struct x86_emulate_ctxt *ctxt)
2758 emulate_2op_SrcV(ctxt, "add");
2759 return X86EMUL_CONTINUE;
2762 static int em_or(struct x86_emulate_ctxt *ctxt)
2764 emulate_2op_SrcV(ctxt, "or");
2765 return X86EMUL_CONTINUE;
2768 static int em_adc(struct x86_emulate_ctxt *ctxt)
2770 emulate_2op_SrcV(ctxt, "adc");
2771 return X86EMUL_CONTINUE;
2774 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2776 emulate_2op_SrcV(ctxt, "sbb");
2777 return X86EMUL_CONTINUE;
2780 static int em_and(struct x86_emulate_ctxt *ctxt)
2782 emulate_2op_SrcV(ctxt, "and");
2783 return X86EMUL_CONTINUE;
2786 static int em_sub(struct x86_emulate_ctxt *ctxt)
2788 emulate_2op_SrcV(ctxt, "sub");
2789 return X86EMUL_CONTINUE;
2792 static int em_xor(struct x86_emulate_ctxt *ctxt)
2794 emulate_2op_SrcV(ctxt, "xor");
2795 return X86EMUL_CONTINUE;
2798 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2800 emulate_2op_SrcV(ctxt, "cmp");
2801 /* Disable writeback. */
2802 ctxt->dst.type = OP_NONE;
2803 return X86EMUL_CONTINUE;
2806 static int em_test(struct x86_emulate_ctxt *ctxt)
2808 emulate_2op_SrcV(ctxt, "test");
2809 /* Disable writeback. */
2810 ctxt->dst.type = OP_NONE;
2811 return X86EMUL_CONTINUE;
2814 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2816 /* Write back the register source. */
2817 ctxt->src.val = ctxt->dst.val;
2818 write_register_operand(&ctxt->src);
2820 /* Write back the memory destination with implicit LOCK prefix. */
2821 ctxt->dst.val = ctxt->src.orig_val;
2822 ctxt->lock_prefix = 1;
2823 return X86EMUL_CONTINUE;
2826 static int em_imul(struct x86_emulate_ctxt *ctxt)
2828 emulate_2op_SrcV_nobyte(ctxt, "imul");
2829 return X86EMUL_CONTINUE;
2832 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2834 ctxt->dst.val = ctxt->src2.val;
2835 return em_imul(ctxt);
2838 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2840 ctxt->dst.type = OP_REG;
2841 ctxt->dst.bytes = ctxt->src.bytes;
2842 ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
2843 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2845 return X86EMUL_CONTINUE;
2848 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2852 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2853 ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
2854 ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
2855 return X86EMUL_CONTINUE;
2858 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
2862 if (ctxt->ops->read_pmc(ctxt, ctxt->regs[VCPU_REGS_RCX], &pmc))
2863 return emulate_gp(ctxt, 0);
2864 ctxt->regs[VCPU_REGS_RAX] = (u32)pmc;
2865 ctxt->regs[VCPU_REGS_RDX] = pmc >> 32;
2866 return X86EMUL_CONTINUE;
2869 static int em_mov(struct x86_emulate_ctxt *ctxt)
2871 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
2872 return X86EMUL_CONTINUE;
2875 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
2877 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
2878 return emulate_gp(ctxt, 0);
2880 /* Disable writeback. */
2881 ctxt->dst.type = OP_NONE;
2882 return X86EMUL_CONTINUE;
2885 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
2889 if (ctxt->mode == X86EMUL_MODE_PROT64)
2890 val = ctxt->src.val & ~0ULL;
2892 val = ctxt->src.val & ~0U;
2894 /* #UD condition is already handled. */
2895 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
2896 return emulate_gp(ctxt, 0);
2898 /* Disable writeback. */
2899 ctxt->dst.type = OP_NONE;
2900 return X86EMUL_CONTINUE;
2903 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
2907 msr_data = (u32)ctxt->regs[VCPU_REGS_RAX]
2908 | ((u64)ctxt->regs[VCPU_REGS_RDX] << 32);
2909 if (ctxt->ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data))
2910 return emulate_gp(ctxt, 0);
2912 return X86EMUL_CONTINUE;
2915 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
2919 if (ctxt->ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data))
2920 return emulate_gp(ctxt, 0);
2922 ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data;
2923 ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32;
2924 return X86EMUL_CONTINUE;
2927 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
2929 if (ctxt->modrm_reg > VCPU_SREG_GS)
2930 return emulate_ud(ctxt);
2932 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
2933 return X86EMUL_CONTINUE;
2936 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
2938 u16 sel = ctxt->src.val;
2940 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
2941 return emulate_ud(ctxt);
2943 if (ctxt->modrm_reg == VCPU_SREG_SS)
2944 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2946 /* Disable writeback. */
2947 ctxt->dst.type = OP_NONE;
2948 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
2951 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2956 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
2957 if (rc == X86EMUL_CONTINUE)
2958 ctxt->ops->invlpg(ctxt, linear);
2959 /* Disable writeback. */
2960 ctxt->dst.type = OP_NONE;
2961 return X86EMUL_CONTINUE;
2964 static int em_clts(struct x86_emulate_ctxt *ctxt)
2968 cr0 = ctxt->ops->get_cr(ctxt, 0);
2970 ctxt->ops->set_cr(ctxt, 0, cr0);
2971 return X86EMUL_CONTINUE;
2974 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
2978 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
2979 return X86EMUL_UNHANDLEABLE;
2981 rc = ctxt->ops->fix_hypercall(ctxt);
2982 if (rc != X86EMUL_CONTINUE)
2985 /* Let the processor re-execute the fixed hypercall */
2986 ctxt->_eip = ctxt->eip;
2987 /* Disable writeback. */
2988 ctxt->dst.type = OP_NONE;
2989 return X86EMUL_CONTINUE;
2992 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
2994 struct desc_ptr desc_ptr;
2997 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2998 &desc_ptr.size, &desc_ptr.address,
3000 if (rc != X86EMUL_CONTINUE)
3002 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3003 /* Disable writeback. */
3004 ctxt->dst.type = OP_NONE;
3005 return X86EMUL_CONTINUE;
3008 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3012 rc = ctxt->ops->fix_hypercall(ctxt);
3014 /* Disable writeback. */
3015 ctxt->dst.type = OP_NONE;
3019 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3021 struct desc_ptr desc_ptr;
3024 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3025 &desc_ptr.size, &desc_ptr.address,
3027 if (rc != X86EMUL_CONTINUE)
3029 ctxt->ops->set_idt(ctxt, &desc_ptr);
3030 /* Disable writeback. */
3031 ctxt->dst.type = OP_NONE;
3032 return X86EMUL_CONTINUE;
3035 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3037 ctxt->dst.bytes = 2;
3038 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3039 return X86EMUL_CONTINUE;
3042 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3044 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3045 | (ctxt->src.val & 0x0f));
3046 ctxt->dst.type = OP_NONE;
3047 return X86EMUL_CONTINUE;
3050 static int em_loop(struct x86_emulate_ctxt *ctxt)
3052 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
3053 if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
3054 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3055 jmp_rel(ctxt, ctxt->src.val);
3057 return X86EMUL_CONTINUE;
3060 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3062 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
3063 jmp_rel(ctxt, ctxt->src.val);
3065 return X86EMUL_CONTINUE;
3068 static int em_in(struct x86_emulate_ctxt *ctxt)
3070 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3072 return X86EMUL_IO_NEEDED;
3074 return X86EMUL_CONTINUE;
3077 static int em_out(struct x86_emulate_ctxt *ctxt)
3079 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3081 /* Disable writeback. */
3082 ctxt->dst.type = OP_NONE;
3083 return X86EMUL_CONTINUE;
3086 static int em_cli(struct x86_emulate_ctxt *ctxt)
3088 if (emulator_bad_iopl(ctxt))
3089 return emulate_gp(ctxt, 0);
3091 ctxt->eflags &= ~X86_EFLAGS_IF;
3092 return X86EMUL_CONTINUE;
3095 static int em_sti(struct x86_emulate_ctxt *ctxt)
3097 if (emulator_bad_iopl(ctxt))
3098 return emulate_gp(ctxt, 0);
3100 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3101 ctxt->eflags |= X86_EFLAGS_IF;
3102 return X86EMUL_CONTINUE;
3105 static int em_bt(struct x86_emulate_ctxt *ctxt)
3107 /* Disable writeback. */
3108 ctxt->dst.type = OP_NONE;
3109 /* only subword offset */
3110 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
3112 emulate_2op_SrcV_nobyte(ctxt, "bt");
3113 return X86EMUL_CONTINUE;
3116 static int em_bts(struct x86_emulate_ctxt *ctxt)
3118 emulate_2op_SrcV_nobyte(ctxt, "bts");
3119 return X86EMUL_CONTINUE;
3122 static int em_btr(struct x86_emulate_ctxt *ctxt)
3124 emulate_2op_SrcV_nobyte(ctxt, "btr");
3125 return X86EMUL_CONTINUE;
3128 static int em_btc(struct x86_emulate_ctxt *ctxt)
3130 emulate_2op_SrcV_nobyte(ctxt, "btc");
3131 return X86EMUL_CONTINUE;
3134 static int em_bsf(struct x86_emulate_ctxt *ctxt)
3136 emulate_2op_SrcV_nobyte(ctxt, "bsf");
3137 return X86EMUL_CONTINUE;
3140 static int em_bsr(struct x86_emulate_ctxt *ctxt)
3142 emulate_2op_SrcV_nobyte(ctxt, "bsr");
3143 return X86EMUL_CONTINUE;
3146 static bool valid_cr(int nr)
3158 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3160 if (!valid_cr(ctxt->modrm_reg))
3161 return emulate_ud(ctxt);
3163 return X86EMUL_CONTINUE;
3166 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3168 u64 new_val = ctxt->src.val64;
3169 int cr = ctxt->modrm_reg;
3172 static u64 cr_reserved_bits[] = {
3173 0xffffffff00000000ULL,
3174 0, 0, 0, /* CR3 checked later */
3181 return emulate_ud(ctxt);
3183 if (new_val & cr_reserved_bits[cr])
3184 return emulate_gp(ctxt, 0);
3189 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3190 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3191 return emulate_gp(ctxt, 0);
3193 cr4 = ctxt->ops->get_cr(ctxt, 4);
3194 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3196 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3197 !(cr4 & X86_CR4_PAE))
3198 return emulate_gp(ctxt, 0);
3205 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3206 if (efer & EFER_LMA)
3207 rsvd = CR3_L_MODE_RESERVED_BITS;
3208 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3209 rsvd = CR3_PAE_RESERVED_BITS;
3210 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3211 rsvd = CR3_NONPAE_RESERVED_BITS;
3214 return emulate_gp(ctxt, 0);
3219 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3221 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3222 return emulate_gp(ctxt, 0);
3228 return X86EMUL_CONTINUE;
3231 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3235 ctxt->ops->get_dr(ctxt, 7, &dr7);
3237 /* Check if DR7.Global_Enable is set */
3238 return dr7 & (1 << 13);
3241 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3243 int dr = ctxt->modrm_reg;
3247 return emulate_ud(ctxt);
3249 cr4 = ctxt->ops->get_cr(ctxt, 4);
3250 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3251 return emulate_ud(ctxt);
3253 if (check_dr7_gd(ctxt))
3254 return emulate_db(ctxt);
3256 return X86EMUL_CONTINUE;
3259 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3261 u64 new_val = ctxt->src.val64;
3262 int dr = ctxt->modrm_reg;
3264 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3265 return emulate_gp(ctxt, 0);
3267 return check_dr_read(ctxt);
3270 static int check_svme(struct x86_emulate_ctxt *ctxt)
3274 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3276 if (!(efer & EFER_SVME))
3277 return emulate_ud(ctxt);
3279 return X86EMUL_CONTINUE;
3282 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3284 u64 rax = ctxt->regs[VCPU_REGS_RAX];
3286 /* Valid physical address? */
3287 if (rax & 0xffff000000000000ULL)
3288 return emulate_gp(ctxt, 0);
3290 return check_svme(ctxt);
3293 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3295 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3297 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3298 return emulate_ud(ctxt);
3300 return X86EMUL_CONTINUE;
3303 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3305 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3306 u64 rcx = ctxt->regs[VCPU_REGS_RCX];
3308 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3310 return emulate_gp(ctxt, 0);
3312 return X86EMUL_CONTINUE;
3315 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3317 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3318 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3319 return emulate_gp(ctxt, 0);
3321 return X86EMUL_CONTINUE;
3324 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3326 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3327 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3328 return emulate_gp(ctxt, 0);
3330 return X86EMUL_CONTINUE;
3333 #define D(_y) { .flags = (_y) }
3334 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3335 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3336 .check_perm = (_p) }
3338 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3339 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3340 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3341 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3342 #define II(_f, _e, _i) \
3343 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3344 #define IIP(_f, _e, _i, _p) \
3345 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3346 .check_perm = (_p) }
3347 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3349 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3350 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3351 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3352 #define I2bvIP(_f, _e, _i, _p) \
3353 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3355 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3356 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3357 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3359 static struct opcode group7_rm1[] = {
3360 DI(SrcNone | Priv, monitor),
3361 DI(SrcNone | Priv, mwait),
3365 static struct opcode group7_rm3[] = {
3366 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3367 II(SrcNone | Prot | VendorSpecific, em_vmmcall, vmmcall),
3368 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3369 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3370 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3371 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3372 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3373 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3376 static struct opcode group7_rm7[] = {
3378 DIP(SrcNone, rdtscp, check_rdtsc),
3382 static struct opcode group1[] = {
3384 I(Lock | PageTable, em_or),
3387 I(Lock | PageTable, em_and),
3393 static struct opcode group1A[] = {
3394 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3397 static struct opcode group3[] = {
3398 I(DstMem | SrcImm, em_test),
3399 I(DstMem | SrcImm, em_test),
3400 I(DstMem | SrcNone | Lock, em_not),
3401 I(DstMem | SrcNone | Lock, em_neg),
3402 I(SrcMem, em_mul_ex),
3403 I(SrcMem, em_imul_ex),
3404 I(SrcMem, em_div_ex),
3405 I(SrcMem, em_idiv_ex),
3408 static struct opcode group4[] = {
3409 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3410 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3414 static struct opcode group5[] = {
3415 I(DstMem | SrcNone | Lock, em_grp45),
3416 I(DstMem | SrcNone | Lock, em_grp45),
3417 I(SrcMem | Stack, em_grp45),
3418 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3419 I(SrcMem | Stack, em_grp45),
3420 I(SrcMemFAddr | ImplicitOps, em_grp45),
3421 I(SrcMem | Stack, em_grp45), N,
3424 static struct opcode group6[] = {
3427 DI(Prot | Priv, lldt),
3428 DI(Prot | Priv, ltr),
3432 static struct group_dual group7 = { {
3433 DI(Mov | DstMem | Priv, sgdt),
3434 DI(Mov | DstMem | Priv, sidt),
3435 II(SrcMem | Priv, em_lgdt, lgdt),
3436 II(SrcMem | Priv, em_lidt, lidt),
3437 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3438 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3439 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3441 I(SrcNone | Priv | VendorSpecific, em_vmcall),
3443 N, EXT(0, group7_rm3),
3444 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3445 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3449 static struct opcode group8[] = {
3451 I(DstMem | SrcImmByte, em_bt),
3452 I(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3453 I(DstMem | SrcImmByte | Lock, em_btr),
3454 I(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3457 static struct group_dual group9 = { {
3458 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3460 N, N, N, N, N, N, N, N,
3463 static struct opcode group11[] = {
3464 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3468 static struct gprefix pfx_0f_6f_0f_7f = {
3469 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3472 static struct gprefix pfx_vmovntpx = {
3473 I(0, em_mov), N, N, N,
3476 static struct opcode opcode_table[256] = {
3478 I6ALU(Lock, em_add),
3479 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3480 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3482 I6ALU(Lock | PageTable, em_or),
3483 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3486 I6ALU(Lock, em_adc),
3487 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3488 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3490 I6ALU(Lock, em_sbb),
3491 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3492 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3494 I6ALU(Lock | PageTable, em_and), N, N,
3496 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3498 I6ALU(Lock, em_xor), N, N,
3500 I6ALU(0, em_cmp), N, N,
3504 X8(I(SrcReg | Stack, em_push)),
3506 X8(I(DstReg | Stack, em_pop)),
3508 I(ImplicitOps | Stack | No64, em_pusha),
3509 I(ImplicitOps | Stack | No64, em_popa),
3510 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3513 I(SrcImm | Mov | Stack, em_push),
3514 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3515 I(SrcImmByte | Mov | Stack, em_push),
3516 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3517 I2bvIP(DstDI | SrcDX | Mov | String, em_in, ins, check_perm_in), /* insb, insw/insd */
3518 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3522 G(ByteOp | DstMem | SrcImm, group1),
3523 G(DstMem | SrcImm, group1),
3524 G(ByteOp | DstMem | SrcImm | No64, group1),
3525 G(DstMem | SrcImmByte, group1),
3526 I2bv(DstMem | SrcReg | ModRM, em_test),
3527 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3529 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3530 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3531 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3532 D(ModRM | SrcMem | NoAccess | DstReg),
3533 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3536 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3538 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3539 I(SrcImmFAddr | No64, em_call_far), N,
3540 II(ImplicitOps | Stack, em_pushf, pushf),
3541 II(ImplicitOps | Stack, em_popf, popf), N, N,
3543 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3544 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3545 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3546 I2bv(SrcSI | DstDI | String, em_cmp),
3548 I2bv(DstAcc | SrcImm, em_test),
3549 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3550 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3551 I2bv(SrcAcc | DstDI | String, em_cmp),
3553 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3555 X8(I(DstReg | SrcImm | Mov, em_mov)),
3557 D2bv(DstMem | SrcImmByte | ModRM),
3558 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3559 I(ImplicitOps | Stack, em_ret),
3560 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3561 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3562 G(ByteOp, group11), G(0, group11),
3564 N, N, N, I(ImplicitOps | Stack, em_ret_far),
3565 D(ImplicitOps), DI(SrcImmByte, intn),
3566 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3568 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3571 N, N, N, N, N, N, N, N,
3573 X3(I(SrcImmByte, em_loop)),
3574 I(SrcImmByte, em_jcxz),
3575 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3576 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3578 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3579 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3580 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3581 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3583 N, DI(ImplicitOps, icebp), N, N,
3584 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3585 G(ByteOp, group3), G(0, group3),
3587 D(ImplicitOps), D(ImplicitOps),
3588 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3589 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3592 static struct opcode twobyte_table[256] = {
3594 G(0, group6), GD(0, &group7), N, N,
3595 N, I(ImplicitOps | VendorSpecific, em_syscall),
3596 II(ImplicitOps | Priv, em_clts, clts), N,
3597 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3598 N, D(ImplicitOps | ModRM), N, N,
3600 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3602 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3603 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3604 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
3605 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
3607 N, N, N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
3610 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3611 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3612 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3613 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3614 I(ImplicitOps | VendorSpecific, em_sysenter),
3615 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3617 N, N, N, N, N, N, N, N,
3619 X16(D(DstReg | SrcMem | ModRM | Mov)),
3621 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3626 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3631 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3635 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3637 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3638 DI(ImplicitOps, cpuid), I(DstMem | SrcReg | ModRM | BitOp, em_bt),
3639 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3640 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3642 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3643 DI(ImplicitOps, rsm),
3644 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3645 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3646 D(DstMem | SrcReg | Src2CL | ModRM),
3647 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3649 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3650 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3651 I(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3652 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3653 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3654 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3658 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3659 I(DstReg | SrcMem | ModRM, em_bsf), I(DstReg | SrcMem | ModRM, em_bsr),
3660 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3662 D2bv(DstMem | SrcReg | ModRM | Lock),
3663 N, D(DstMem | SrcReg | ModRM | Mov),
3664 N, N, N, GD(0, &group9),
3665 N, N, N, N, N, N, N, N,
3667 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3669 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3671 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3688 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3692 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3698 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3699 unsigned size, bool sign_extension)
3701 int rc = X86EMUL_CONTINUE;
3705 op->addr.mem.ea = ctxt->_eip;
3706 /* NB. Immediates are sign-extended as necessary. */
3707 switch (op->bytes) {
3709 op->val = insn_fetch(s8, ctxt);
3712 op->val = insn_fetch(s16, ctxt);
3715 op->val = insn_fetch(s32, ctxt);
3718 if (!sign_extension) {
3719 switch (op->bytes) {
3727 op->val &= 0xffffffff;
3735 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
3738 int rc = X86EMUL_CONTINUE;
3742 decode_register_operand(ctxt, op);
3745 rc = decode_imm(ctxt, op, 1, false);
3748 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3752 if ((ctxt->d & BitOp) && op == &ctxt->dst)
3753 fetch_bit_operand(ctxt);
3754 op->orig_val = op->val;
3757 ctxt->memop.bytes = 8;
3761 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3762 op->addr.reg = &ctxt->regs[VCPU_REGS_RAX];
3763 fetch_register_operand(op);
3764 op->orig_val = op->val;
3768 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3770 register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]);
3771 op->addr.mem.seg = VCPU_SREG_ES;
3777 op->addr.reg = &ctxt->regs[VCPU_REGS_RDX];
3778 fetch_register_operand(op);
3782 op->val = ctxt->regs[VCPU_REGS_RCX] & 0xff;
3785 rc = decode_imm(ctxt, op, 1, true);
3792 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
3795 ctxt->memop.bytes = 1;
3798 ctxt->memop.bytes = 2;
3801 ctxt->memop.bytes = 4;
3804 rc = decode_imm(ctxt, op, 2, false);
3807 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
3811 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3813 register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]);
3814 op->addr.mem.seg = seg_override(ctxt);
3819 op->addr.mem.ea = ctxt->_eip;
3820 op->bytes = ctxt->op_bytes + 2;
3821 insn_fetch_arr(op->valptr, op->bytes, ctxt);
3824 ctxt->memop.bytes = ctxt->op_bytes + 2;
3827 op->val = VCPU_SREG_ES;
3830 op->val = VCPU_SREG_CS;
3833 op->val = VCPU_SREG_SS;
3836 op->val = VCPU_SREG_DS;
3839 op->val = VCPU_SREG_FS;
3842 op->val = VCPU_SREG_GS;
3845 /* Special instructions do their own operand decoding. */
3847 op->type = OP_NONE; /* Disable writeback. */
3855 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3857 int rc = X86EMUL_CONTINUE;
3858 int mode = ctxt->mode;
3859 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3860 bool op_prefix = false;
3861 struct opcode opcode;
3863 ctxt->memop.type = OP_NONE;
3864 ctxt->memopp = NULL;
3865 ctxt->_eip = ctxt->eip;
3866 ctxt->fetch.start = ctxt->_eip;
3867 ctxt->fetch.end = ctxt->fetch.start + insn_len;
3869 memcpy(ctxt->fetch.data, insn, insn_len);
3872 case X86EMUL_MODE_REAL:
3873 case X86EMUL_MODE_VM86:
3874 case X86EMUL_MODE_PROT16:
3875 def_op_bytes = def_ad_bytes = 2;
3877 case X86EMUL_MODE_PROT32:
3878 def_op_bytes = def_ad_bytes = 4;
3880 #ifdef CONFIG_X86_64
3881 case X86EMUL_MODE_PROT64:
3887 return EMULATION_FAILED;
3890 ctxt->op_bytes = def_op_bytes;
3891 ctxt->ad_bytes = def_ad_bytes;
3893 /* Legacy prefixes. */
3895 switch (ctxt->b = insn_fetch(u8, ctxt)) {
3896 case 0x66: /* operand-size override */
3898 /* switch between 2/4 bytes */
3899 ctxt->op_bytes = def_op_bytes ^ 6;
3901 case 0x67: /* address-size override */
3902 if (mode == X86EMUL_MODE_PROT64)
3903 /* switch between 4/8 bytes */
3904 ctxt->ad_bytes = def_ad_bytes ^ 12;
3906 /* switch between 2/4 bytes */
3907 ctxt->ad_bytes = def_ad_bytes ^ 6;
3909 case 0x26: /* ES override */
3910 case 0x2e: /* CS override */
3911 case 0x36: /* SS override */
3912 case 0x3e: /* DS override */
3913 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
3915 case 0x64: /* FS override */
3916 case 0x65: /* GS override */
3917 set_seg_override(ctxt, ctxt->b & 7);
3919 case 0x40 ... 0x4f: /* REX */
3920 if (mode != X86EMUL_MODE_PROT64)
3922 ctxt->rex_prefix = ctxt->b;
3924 case 0xf0: /* LOCK */
3925 ctxt->lock_prefix = 1;
3927 case 0xf2: /* REPNE/REPNZ */
3928 case 0xf3: /* REP/REPE/REPZ */
3929 ctxt->rep_prefix = ctxt->b;
3935 /* Any legacy prefix after a REX prefix nullifies its effect. */
3937 ctxt->rex_prefix = 0;
3943 if (ctxt->rex_prefix & 8)
3944 ctxt->op_bytes = 8; /* REX.W */
3946 /* Opcode byte(s). */
3947 opcode = opcode_table[ctxt->b];
3948 /* Two-byte opcode? */
3949 if (ctxt->b == 0x0f) {
3951 ctxt->b = insn_fetch(u8, ctxt);
3952 opcode = twobyte_table[ctxt->b];
3954 ctxt->d = opcode.flags;
3956 if (ctxt->d & ModRM)
3957 ctxt->modrm = insn_fetch(u8, ctxt);
3959 while (ctxt->d & GroupMask) {
3960 switch (ctxt->d & GroupMask) {
3962 goffset = (ctxt->modrm >> 3) & 7;
3963 opcode = opcode.u.group[goffset];
3966 goffset = (ctxt->modrm >> 3) & 7;
3967 if ((ctxt->modrm >> 6) == 3)
3968 opcode = opcode.u.gdual->mod3[goffset];
3970 opcode = opcode.u.gdual->mod012[goffset];
3973 goffset = ctxt->modrm & 7;
3974 opcode = opcode.u.group[goffset];
3977 if (ctxt->rep_prefix && op_prefix)
3978 return EMULATION_FAILED;
3979 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
3980 switch (simd_prefix) {
3981 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3982 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3983 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3984 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3988 return EMULATION_FAILED;
3991 ctxt->d &= ~(u64)GroupMask;
3992 ctxt->d |= opcode.flags;
3995 ctxt->execute = opcode.u.execute;
3996 ctxt->check_perm = opcode.check_perm;
3997 ctxt->intercept = opcode.intercept;
4000 if (ctxt->d == 0 || (ctxt->d & Undefined))
4001 return EMULATION_FAILED;
4003 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
4004 return EMULATION_FAILED;
4006 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4009 if (ctxt->d & Op3264) {
4010 if (mode == X86EMUL_MODE_PROT64)
4017 ctxt->op_bytes = 16;
4018 else if (ctxt->d & Mmx)
4021 /* ModRM and SIB bytes. */
4022 if (ctxt->d & ModRM) {
4023 rc = decode_modrm(ctxt, &ctxt->memop);
4024 if (!ctxt->has_seg_override)
4025 set_seg_override(ctxt, ctxt->modrm_seg);
4026 } else if (ctxt->d & MemAbs)
4027 rc = decode_abs(ctxt, &ctxt->memop);
4028 if (rc != X86EMUL_CONTINUE)
4031 if (!ctxt->has_seg_override)
4032 set_seg_override(ctxt, VCPU_SREG_DS);
4034 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4036 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4037 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4040 * Decode and fetch the source operand: register, memory
4043 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4044 if (rc != X86EMUL_CONTINUE)
4048 * Decode and fetch the second source operand: register, memory
4051 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4052 if (rc != X86EMUL_CONTINUE)
4055 /* Decode and fetch the destination operand: register or memory. */
4056 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4059 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4060 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4062 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4065 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4067 return ctxt->d & PageTable;
4070 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4072 /* The second termination condition only applies for REPE
4073 * and REPNE. Test if the repeat string operation prefix is
4074 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4075 * corresponding termination condition according to:
4076 * - if REPE/REPZ and ZF = 0 then done
4077 * - if REPNE/REPNZ and ZF = 1 then done
4079 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4080 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4081 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4082 ((ctxt->eflags & EFLG_ZF) == 0))
4083 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4084 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4090 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4094 ctxt->ops->get_fpu(ctxt);
4095 asm volatile("1: fwait \n\t"
4097 ".pushsection .fixup,\"ax\" \n\t"
4099 "movb $1, %[fault] \n\t"
4102 _ASM_EXTABLE(1b, 3b)
4103 : [fault]"+qm"(fault));
4104 ctxt->ops->put_fpu(ctxt);
4106 if (unlikely(fault))
4107 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4109 return X86EMUL_CONTINUE;
4112 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4115 if (op->type == OP_MM)
4116 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4119 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4121 struct x86_emulate_ops *ops = ctxt->ops;
4122 int rc = X86EMUL_CONTINUE;
4123 int saved_dst_type = ctxt->dst.type;
4125 ctxt->mem_read.pos = 0;
4127 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
4128 rc = emulate_ud(ctxt);
4132 /* LOCK prefix is allowed only with some instructions */
4133 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4134 rc = emulate_ud(ctxt);
4138 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4139 rc = emulate_ud(ctxt);
4143 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4144 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4145 rc = emulate_ud(ctxt);
4149 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4150 rc = emulate_nm(ctxt);
4154 if (ctxt->d & Mmx) {
4155 rc = flush_pending_x87_faults(ctxt);
4156 if (rc != X86EMUL_CONTINUE)
4159 * Now that we know the fpu is exception safe, we can fetch
4162 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4163 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4164 if (!(ctxt->d & Mov))
4165 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4168 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4169 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4170 X86_ICPT_PRE_EXCEPT);
4171 if (rc != X86EMUL_CONTINUE)
4175 /* Privileged instruction can be executed only in CPL=0 */
4176 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4177 rc = emulate_gp(ctxt, 0);
4181 /* Instruction can only be executed in protected mode */
4182 if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
4183 rc = emulate_ud(ctxt);
4187 /* Do instruction specific permission checks */
4188 if (ctxt->check_perm) {
4189 rc = ctxt->check_perm(ctxt);
4190 if (rc != X86EMUL_CONTINUE)
4194 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4195 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4196 X86_ICPT_POST_EXCEPT);
4197 if (rc != X86EMUL_CONTINUE)
4201 if (ctxt->rep_prefix && (ctxt->d & String)) {
4202 /* All REP prefixes have the same first termination condition */
4203 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) {
4204 ctxt->eip = ctxt->_eip;
4209 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4210 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4211 ctxt->src.valptr, ctxt->src.bytes);
4212 if (rc != X86EMUL_CONTINUE)
4214 ctxt->src.orig_val64 = ctxt->src.val64;
4217 if (ctxt->src2.type == OP_MEM) {
4218 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4219 &ctxt->src2.val, ctxt->src2.bytes);
4220 if (rc != X86EMUL_CONTINUE)
4224 if ((ctxt->d & DstMask) == ImplicitOps)
4228 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4229 /* optimisation - avoid slow emulated read if Mov */
4230 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4231 &ctxt->dst.val, ctxt->dst.bytes);
4232 if (rc != X86EMUL_CONTINUE)
4235 ctxt->dst.orig_val = ctxt->dst.val;
4239 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4240 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4241 X86_ICPT_POST_MEMACCESS);
4242 if (rc != X86EMUL_CONTINUE)
4246 if (ctxt->execute) {
4247 rc = ctxt->execute(ctxt);
4248 if (rc != X86EMUL_CONTINUE)
4257 case 0x40 ... 0x47: /* inc r16/r32 */
4258 emulate_1op(ctxt, "inc");
4260 case 0x48 ... 0x4f: /* dec r16/r32 */
4261 emulate_1op(ctxt, "dec");
4263 case 0x63: /* movsxd */
4264 if (ctxt->mode != X86EMUL_MODE_PROT64)
4265 goto cannot_emulate;
4266 ctxt->dst.val = (s32) ctxt->src.val;
4268 case 0x70 ... 0x7f: /* jcc (short) */
4269 if (test_cc(ctxt->b, ctxt->eflags))
4270 jmp_rel(ctxt, ctxt->src.val);
4272 case 0x8d: /* lea r16/r32, m */
4273 ctxt->dst.val = ctxt->src.addr.mem.ea;
4275 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4276 if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX])
4280 case 0x98: /* cbw/cwde/cdqe */
4281 switch (ctxt->op_bytes) {
4282 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4283 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4284 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4290 case 0xcc: /* int3 */
4291 rc = emulate_int(ctxt, 3);
4293 case 0xcd: /* int n */
4294 rc = emulate_int(ctxt, ctxt->src.val);
4296 case 0xce: /* into */
4297 if (ctxt->eflags & EFLG_OF)
4298 rc = emulate_int(ctxt, 4);
4300 case 0xd0 ... 0xd1: /* Grp2 */
4303 case 0xd2 ... 0xd3: /* Grp2 */
4304 ctxt->src.val = ctxt->regs[VCPU_REGS_RCX];
4307 case 0xe9: /* jmp rel */
4308 case 0xeb: /* jmp rel short */
4309 jmp_rel(ctxt, ctxt->src.val);
4310 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4312 case 0xf4: /* hlt */
4313 ctxt->ops->halt(ctxt);
4315 case 0xf5: /* cmc */
4316 /* complement carry flag from eflags reg */
4317 ctxt->eflags ^= EFLG_CF;
4319 case 0xf8: /* clc */
4320 ctxt->eflags &= ~EFLG_CF;
4322 case 0xf9: /* stc */
4323 ctxt->eflags |= EFLG_CF;
4325 case 0xfc: /* cld */
4326 ctxt->eflags &= ~EFLG_DF;
4328 case 0xfd: /* std */
4329 ctxt->eflags |= EFLG_DF;
4332 goto cannot_emulate;
4335 if (rc != X86EMUL_CONTINUE)
4339 rc = writeback(ctxt);
4340 if (rc != X86EMUL_CONTINUE)
4344 * restore dst type in case the decoding will be reused
4345 * (happens for string instruction )
4347 ctxt->dst.type = saved_dst_type;
4349 if ((ctxt->d & SrcMask) == SrcSI)
4350 string_addr_inc(ctxt, seg_override(ctxt),
4351 VCPU_REGS_RSI, &ctxt->src);
4353 if ((ctxt->d & DstMask) == DstDI)
4354 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4357 if (ctxt->rep_prefix && (ctxt->d & String)) {
4358 struct read_cache *r = &ctxt->io_read;
4359 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
4361 if (!string_insn_completed(ctxt)) {
4363 * Re-enter guest when pio read ahead buffer is empty
4364 * or, if it is not used, after each 1024 iteration.
4366 if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) &&
4367 (r->end == 0 || r->end != r->pos)) {
4369 * Reset read cache. Usually happens before
4370 * decode, but since instruction is restarted
4371 * we have to do it here.
4373 ctxt->mem_read.end = 0;
4374 return EMULATION_RESTART;
4376 goto done; /* skip rip writeback */
4380 ctxt->eip = ctxt->_eip;
4383 if (rc == X86EMUL_PROPAGATE_FAULT)
4384 ctxt->have_exception = true;
4385 if (rc == X86EMUL_INTERCEPTED)
4386 return EMULATION_INTERCEPTED;
4388 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4392 case 0x09: /* wbinvd */
4393 (ctxt->ops->wbinvd)(ctxt);
4395 case 0x08: /* invd */
4396 case 0x0d: /* GrpP (prefetch) */
4397 case 0x18: /* Grp16 (prefetch/nop) */
4399 case 0x20: /* mov cr, reg */
4400 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4402 case 0x21: /* mov from dr to reg */
4403 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4405 case 0x40 ... 0x4f: /* cmov */
4406 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4407 if (!test_cc(ctxt->b, ctxt->eflags))
4408 ctxt->dst.type = OP_NONE; /* no writeback */
4410 case 0x80 ... 0x8f: /* jnz rel, etc*/
4411 if (test_cc(ctxt->b, ctxt->eflags))
4412 jmp_rel(ctxt, ctxt->src.val);
4414 case 0x90 ... 0x9f: /* setcc r/m8 */
4415 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4417 case 0xa4: /* shld imm8, r, r/m */
4418 case 0xa5: /* shld cl, r, r/m */
4419 emulate_2op_cl(ctxt, "shld");
4421 case 0xac: /* shrd imm8, r, r/m */
4422 case 0xad: /* shrd cl, r, r/m */
4423 emulate_2op_cl(ctxt, "shrd");
4425 case 0xae: /* clflush */
4427 case 0xb6 ... 0xb7: /* movzx */
4428 ctxt->dst.bytes = ctxt->op_bytes;
4429 ctxt->dst.val = (ctxt->d & ByteOp) ? (u8) ctxt->src.val
4430 : (u16) ctxt->src.val;
4432 case 0xbe ... 0xbf: /* movsx */
4433 ctxt->dst.bytes = ctxt->op_bytes;
4434 ctxt->dst.val = (ctxt->d & ByteOp) ? (s8) ctxt->src.val :
4435 (s16) ctxt->src.val;
4437 case 0xc0 ... 0xc1: /* xadd */
4438 emulate_2op_SrcV(ctxt, "add");
4439 /* Write back the register source. */
4440 ctxt->src.val = ctxt->dst.orig_val;
4441 write_register_operand(&ctxt->src);
4443 case 0xc3: /* movnti */
4444 ctxt->dst.bytes = ctxt->op_bytes;
4445 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4446 (u64) ctxt->src.val;
4449 goto cannot_emulate;
4452 if (rc != X86EMUL_CONTINUE)
4458 return EMULATION_FAILED;