1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
35 #define OpImplicit 1ull /* No generic decode */
36 #define OpReg 2ull /* Register */
37 #define OpMem 3ull /* Memory */
38 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
39 #define OpDI 5ull /* ES:DI/EDI/RDI */
40 #define OpMem64 6ull /* Memory, 64-bit */
41 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
42 #define OpDX 8ull /* DX register */
43 #define OpCL 9ull /* CL register (for shifts) */
44 #define OpImmByte 10ull /* 8-bit sign extended immediate */
45 #define OpOne 11ull /* Implied 1 */
46 #define OpImm 12ull /* Sign extended immediate */
47 #define OpMem16 13ull /* Memory operand (16-bit). */
48 #define OpMem32 14ull /* Memory operand (32-bit). */
49 #define OpImmU 15ull /* Immediate operand, zero extended */
50 #define OpSI 16ull /* SI/ESI/RSI */
51 #define OpImmFAddr 17ull /* Immediate far address */
52 #define OpMemFAddr 18ull /* Far address in memory */
53 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
54 #define OpES 20ull /* ES */
55 #define OpCS 21ull /* CS */
56 #define OpSS 22ull /* SS */
57 #define OpDS 23ull /* DS */
58 #define OpFS 24ull /* FS */
59 #define OpGS 25ull /* GS */
61 #define OpBits 5 /* Width of operand field */
62 #define OpMask ((1ull << OpBits) - 1)
65 * Opcode effective-address decode tables.
66 * Note that we only emulate instructions that have at least one memory
67 * operand (excluding implicit stack references). We assume that stack
68 * references and instruction fetches will never occur in special memory
69 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
73 /* Operand sizes: 8-bit operands or specified/overridden size. */
74 #define ByteOp (1<<0) /* 8-bit operands. */
75 /* Destination operand type. */
77 #define ImplicitOps (OpImplicit << DstShift)
78 #define DstReg (OpReg << DstShift)
79 #define DstMem (OpMem << DstShift)
80 #define DstAcc (OpAcc << DstShift)
81 #define DstDI (OpDI << DstShift)
82 #define DstMem64 (OpMem64 << DstShift)
83 #define DstImmUByte (OpImmUByte << DstShift)
84 #define DstDX (OpDX << DstShift)
85 #define DstMask (OpMask << DstShift)
86 /* Source operand type. */
88 #define SrcNone (OpNone << SrcShift)
89 #define SrcReg (OpReg << SrcShift)
90 #define SrcMem (OpMem << SrcShift)
91 #define SrcMem16 (OpMem16 << SrcShift)
92 #define SrcMem32 (OpMem32 << SrcShift)
93 #define SrcImm (OpImm << SrcShift)
94 #define SrcImmByte (OpImmByte << SrcShift)
95 #define SrcOne (OpOne << SrcShift)
96 #define SrcImmUByte (OpImmUByte << SrcShift)
97 #define SrcImmU (OpImmU << SrcShift)
98 #define SrcSI (OpSI << SrcShift)
99 #define SrcImmFAddr (OpImmFAddr << SrcShift)
100 #define SrcMemFAddr (OpMemFAddr << SrcShift)
101 #define SrcAcc (OpAcc << SrcShift)
102 #define SrcImmU16 (OpImmU16 << SrcShift)
103 #define SrcDX (OpDX << SrcShift)
104 #define SrcMask (OpMask << SrcShift)
105 #define BitOp (1<<11)
106 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
107 #define String (1<<13) /* String instruction (rep capable) */
108 #define Stack (1<<14) /* Stack instruction (push/pop) */
109 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
110 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
111 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
112 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
113 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
114 #define Sse (1<<18) /* SSE Vector instruction */
115 /* Generic ModRM decode. */
116 #define ModRM (1<<19)
117 /* Destination is only written; never read. */
120 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
121 #define VendorSpecific (1<<22) /* Vendor specific instruction */
122 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
123 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
124 #define Undefined (1<<25) /* No Such Instruction */
125 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
126 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
128 /* Source 2 operand type */
129 #define Src2Shift (29)
130 #define Src2None (OpNone << Src2Shift)
131 #define Src2CL (OpCL << Src2Shift)
132 #define Src2ImmByte (OpImmByte << Src2Shift)
133 #define Src2One (OpOne << Src2Shift)
134 #define Src2Imm (OpImm << Src2Shift)
135 #define Src2ES (OpES << Src2Shift)
136 #define Src2CS (OpCS << Src2Shift)
137 #define Src2SS (OpSS << Src2Shift)
138 #define Src2DS (OpDS << Src2Shift)
139 #define Src2FS (OpFS << Src2Shift)
140 #define Src2GS (OpGS << Src2Shift)
141 #define Src2Mask (OpMask << Src2Shift)
143 #define X2(x...) x, x
144 #define X3(x...) X2(x), x
145 #define X4(x...) X2(x), X2(x)
146 #define X5(x...) X4(x), x
147 #define X6(x...) X4(x), X2(x)
148 #define X7(x...) X4(x), X3(x)
149 #define X8(x...) X4(x), X4(x)
150 #define X16(x...) X8(x), X8(x)
156 int (*execute)(struct x86_emulate_ctxt *ctxt);
157 struct opcode *group;
158 struct group_dual *gdual;
159 struct gprefix *gprefix;
161 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
165 struct opcode mod012[8];
166 struct opcode mod3[8];
170 struct opcode pfx_no;
171 struct opcode pfx_66;
172 struct opcode pfx_f2;
173 struct opcode pfx_f3;
176 /* EFLAGS bit definitions. */
177 #define EFLG_ID (1<<21)
178 #define EFLG_VIP (1<<20)
179 #define EFLG_VIF (1<<19)
180 #define EFLG_AC (1<<18)
181 #define EFLG_VM (1<<17)
182 #define EFLG_RF (1<<16)
183 #define EFLG_IOPL (3<<12)
184 #define EFLG_NT (1<<14)
185 #define EFLG_OF (1<<11)
186 #define EFLG_DF (1<<10)
187 #define EFLG_IF (1<<9)
188 #define EFLG_TF (1<<8)
189 #define EFLG_SF (1<<7)
190 #define EFLG_ZF (1<<6)
191 #define EFLG_AF (1<<4)
192 #define EFLG_PF (1<<2)
193 #define EFLG_CF (1<<0)
195 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
196 #define EFLG_RESERVED_ONE_MASK 2
199 * Instruction emulation:
200 * Most instructions are emulated directly via a fragment of inline assembly
201 * code. This allows us to save/restore EFLAGS and thus very easily pick up
202 * any modified flags.
205 #if defined(CONFIG_X86_64)
206 #define _LO32 "k" /* force 32-bit operand */
207 #define _STK "%%rsp" /* stack pointer */
208 #elif defined(__i386__)
209 #define _LO32 "" /* force 32-bit operand */
210 #define _STK "%%esp" /* stack pointer */
214 * These EFLAGS bits are restored from saved value during emulation, and
215 * any changes are written back to the saved value after emulation.
217 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
219 /* Before executing instruction: restore necessary bits in EFLAGS. */
220 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
221 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
222 "movl %"_sav",%"_LO32 _tmp"; " \
225 "movl %"_msk",%"_LO32 _tmp"; " \
226 "andl %"_LO32 _tmp",("_STK"); " \
228 "notl %"_LO32 _tmp"; " \
229 "andl %"_LO32 _tmp",("_STK"); " \
230 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
232 "orl %"_LO32 _tmp",("_STK"); " \
236 /* After executing instruction: write-back necessary bits in EFLAGS. */
237 #define _POST_EFLAGS(_sav, _msk, _tmp) \
238 /* _sav |= EFLAGS & _msk; */ \
241 "andl %"_msk",%"_LO32 _tmp"; " \
242 "orl %"_LO32 _tmp",%"_sav"; "
250 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
252 __asm__ __volatile__ ( \
253 _PRE_EFLAGS("0", "4", "2") \
254 _op _suffix " %"_x"3,%1; " \
255 _POST_EFLAGS("0", "4", "2") \
256 : "=m" ((ctxt)->eflags), \
257 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
259 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
263 /* Raw emulation: instruction has two explicit operands. */
264 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
266 unsigned long _tmp; \
268 switch ((ctxt)->dst.bytes) { \
270 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
273 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
276 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
281 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
283 unsigned long _tmp; \
284 switch ((ctxt)->dst.bytes) { \
286 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
289 __emulate_2op_nobyte(ctxt, _op, \
290 _wx, _wy, _lx, _ly, _qx, _qy); \
295 /* Source operand is byte-sized and may be restricted to just %cl. */
296 #define emulate_2op_SrcB(ctxt, _op) \
297 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
299 /* Source operand is byte, word, long or quad sized. */
300 #define emulate_2op_SrcV(ctxt, _op) \
301 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
303 /* Source operand is word, long or quad sized. */
304 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
305 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
307 /* Instruction has three operands and one operand is stored in ECX register */
308 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
310 unsigned long _tmp; \
311 _type _clv = (ctxt)->src2.val; \
312 _type _srcv = (ctxt)->src.val; \
313 _type _dstv = (ctxt)->dst.val; \
315 __asm__ __volatile__ ( \
316 _PRE_EFLAGS("0", "5", "2") \
317 _op _suffix " %4,%1 \n" \
318 _POST_EFLAGS("0", "5", "2") \
319 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
320 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
323 (ctxt)->src2.val = (unsigned long) _clv; \
324 (ctxt)->src2.val = (unsigned long) _srcv; \
325 (ctxt)->dst.val = (unsigned long) _dstv; \
328 #define emulate_2op_cl(ctxt, _op) \
330 switch ((ctxt)->dst.bytes) { \
332 __emulate_2op_cl(ctxt, _op, "w", u16); \
335 __emulate_2op_cl(ctxt, _op, "l", u32); \
338 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
343 #define __emulate_1op(ctxt, _op, _suffix) \
345 unsigned long _tmp; \
347 __asm__ __volatile__ ( \
348 _PRE_EFLAGS("0", "3", "2") \
349 _op _suffix " %1; " \
350 _POST_EFLAGS("0", "3", "2") \
351 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
353 : "i" (EFLAGS_MASK)); \
356 /* Instruction has only one explicit operand (no source operand). */
357 #define emulate_1op(ctxt, _op) \
359 switch ((ctxt)->dst.bytes) { \
360 case 1: __emulate_1op(ctxt, _op, "b"); break; \
361 case 2: __emulate_1op(ctxt, _op, "w"); break; \
362 case 4: __emulate_1op(ctxt, _op, "l"); break; \
363 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
367 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
369 unsigned long _tmp; \
370 ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX]; \
371 ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX]; \
373 __asm__ __volatile__ ( \
374 _PRE_EFLAGS("0", "5", "1") \
376 _op _suffix " %6; " \
378 _POST_EFLAGS("0", "5", "1") \
379 ".pushsection .fixup,\"ax\" \n\t" \
380 "3: movb $1, %4 \n\t" \
383 _ASM_EXTABLE(1b, 3b) \
384 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
385 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
386 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val), \
387 "a" (*rax), "d" (*rdx)); \
390 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
391 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
393 switch((ctxt)->src.bytes) { \
395 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
398 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
401 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
404 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
409 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
410 enum x86_intercept intercept,
411 enum x86_intercept_stage stage)
413 struct x86_instruction_info info = {
414 .intercept = intercept,
415 .rep_prefix = ctxt->rep_prefix,
416 .modrm_mod = ctxt->modrm_mod,
417 .modrm_reg = ctxt->modrm_reg,
418 .modrm_rm = ctxt->modrm_rm,
419 .src_val = ctxt->src.val64,
420 .src_bytes = ctxt->src.bytes,
421 .dst_bytes = ctxt->dst.bytes,
422 .ad_bytes = ctxt->ad_bytes,
423 .next_rip = ctxt->eip,
426 return ctxt->ops->intercept(ctxt, &info, stage);
429 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
431 return (1UL << (ctxt->ad_bytes << 3)) - 1;
434 /* Access/update address held in a register, based on addressing mode. */
435 static inline unsigned long
436 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
438 if (ctxt->ad_bytes == sizeof(unsigned long))
441 return reg & ad_mask(ctxt);
444 static inline unsigned long
445 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
447 return address_mask(ctxt, reg);
451 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
453 if (ctxt->ad_bytes == sizeof(unsigned long))
456 *reg = (*reg & ~ad_mask(ctxt)) | ((*reg + inc) & ad_mask(ctxt));
459 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
461 register_address_increment(ctxt, &ctxt->_eip, rel);
464 static u32 desc_limit_scaled(struct desc_struct *desc)
466 u32 limit = get_desc_limit(desc);
468 return desc->g ? (limit << 12) | 0xfff : limit;
471 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
473 ctxt->has_seg_override = true;
474 ctxt->seg_override = seg;
477 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
479 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
482 return ctxt->ops->get_cached_segment_base(ctxt, seg);
485 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
487 if (!ctxt->has_seg_override)
490 return ctxt->seg_override;
493 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
494 u32 error, bool valid)
496 ctxt->exception.vector = vec;
497 ctxt->exception.error_code = error;
498 ctxt->exception.error_code_valid = valid;
499 return X86EMUL_PROPAGATE_FAULT;
502 static int emulate_db(struct x86_emulate_ctxt *ctxt)
504 return emulate_exception(ctxt, DB_VECTOR, 0, false);
507 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
509 return emulate_exception(ctxt, GP_VECTOR, err, true);
512 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
514 return emulate_exception(ctxt, SS_VECTOR, err, true);
517 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
519 return emulate_exception(ctxt, UD_VECTOR, 0, false);
522 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
524 return emulate_exception(ctxt, TS_VECTOR, err, true);
527 static int emulate_de(struct x86_emulate_ctxt *ctxt)
529 return emulate_exception(ctxt, DE_VECTOR, 0, false);
532 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
534 return emulate_exception(ctxt, NM_VECTOR, 0, false);
537 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
540 struct desc_struct desc;
542 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
546 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
551 struct desc_struct desc;
553 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
554 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
557 static int __linearize(struct x86_emulate_ctxt *ctxt,
558 struct segmented_address addr,
559 unsigned size, bool write, bool fetch,
562 struct desc_struct desc;
569 la = seg_base(ctxt, addr.seg) + addr.ea;
570 switch (ctxt->mode) {
571 case X86EMUL_MODE_REAL:
573 case X86EMUL_MODE_PROT64:
574 if (((signed long)la << 16) >> 16 != la)
575 return emulate_gp(ctxt, 0);
578 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
582 /* code segment or read-only data segment */
583 if (((desc.type & 8) || !(desc.type & 2)) && write)
585 /* unreadable code segment */
586 if (!fetch && (desc.type & 8) && !(desc.type & 2))
588 lim = desc_limit_scaled(&desc);
589 if ((desc.type & 8) || !(desc.type & 4)) {
590 /* expand-up segment */
591 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
594 /* exapand-down segment */
595 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
597 lim = desc.d ? 0xffffffff : 0xffff;
598 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
601 cpl = ctxt->ops->cpl(ctxt);
604 if (!(desc.type & 8)) {
608 } else if ((desc.type & 8) && !(desc.type & 4)) {
609 /* nonconforming code segment */
612 } else if ((desc.type & 8) && (desc.type & 4)) {
613 /* conforming code segment */
619 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
622 return X86EMUL_CONTINUE;
624 if (addr.seg == VCPU_SREG_SS)
625 return emulate_ss(ctxt, addr.seg);
627 return emulate_gp(ctxt, addr.seg);
630 static int linearize(struct x86_emulate_ctxt *ctxt,
631 struct segmented_address addr,
632 unsigned size, bool write,
635 return __linearize(ctxt, addr, size, write, false, linear);
639 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
640 struct segmented_address addr,
647 rc = linearize(ctxt, addr, size, false, &linear);
648 if (rc != X86EMUL_CONTINUE)
650 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
654 * Fetch the next byte of the instruction being emulated which is pointed to
655 * by ctxt->_eip, then increment ctxt->_eip.
657 * Also prefetch the remaining bytes of the instruction without crossing page
658 * boundary if they are not in fetch_cache yet.
660 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
662 struct fetch_cache *fc = &ctxt->fetch;
666 if (ctxt->_eip == fc->end) {
667 unsigned long linear;
668 struct segmented_address addr = { .seg = VCPU_SREG_CS,
670 cur_size = fc->end - fc->start;
671 size = min(15UL - cur_size,
672 PAGE_SIZE - offset_in_page(ctxt->_eip));
673 rc = __linearize(ctxt, addr, size, false, true, &linear);
674 if (unlikely(rc != X86EMUL_CONTINUE))
676 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
677 size, &ctxt->exception);
678 if (unlikely(rc != X86EMUL_CONTINUE))
682 *dest = fc->data[ctxt->_eip - fc->start];
684 return X86EMUL_CONTINUE;
687 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
688 void *dest, unsigned size)
692 /* x86 instructions are limited to 15 bytes. */
693 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
694 return X86EMUL_UNHANDLEABLE;
696 rc = do_insn_fetch_byte(ctxt, dest++);
697 if (rc != X86EMUL_CONTINUE)
700 return X86EMUL_CONTINUE;
703 /* Fetch next part of the instruction being emulated. */
704 #define insn_fetch(_type, _ctxt) \
705 ({ unsigned long _x; \
706 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
707 if (rc != X86EMUL_CONTINUE) \
712 #define insn_fetch_arr(_arr, _size, _ctxt) \
713 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
714 if (rc != X86EMUL_CONTINUE) \
719 * Given the 'reg' portion of a ModRM byte, and a register block, return a
720 * pointer into the block that addresses the relevant register.
721 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
723 static void *decode_register(u8 modrm_reg, unsigned long *regs,
728 p = ®s[modrm_reg];
729 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
730 p = (unsigned char *)®s[modrm_reg & 3] + 1;
734 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
735 struct segmented_address addr,
736 u16 *size, unsigned long *address, int op_bytes)
743 rc = segmented_read_std(ctxt, addr, size, 2);
744 if (rc != X86EMUL_CONTINUE)
747 rc = segmented_read_std(ctxt, addr, address, op_bytes);
751 static int test_cc(unsigned int condition, unsigned int flags)
755 switch ((condition & 15) >> 1) {
757 rc |= (flags & EFLG_OF);
759 case 1: /* b/c/nae */
760 rc |= (flags & EFLG_CF);
763 rc |= (flags & EFLG_ZF);
766 rc |= (flags & (EFLG_CF|EFLG_ZF));
769 rc |= (flags & EFLG_SF);
772 rc |= (flags & EFLG_PF);
775 rc |= (flags & EFLG_ZF);
778 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
782 /* Odd condition identifiers (lsb == 1) have inverted sense. */
783 return (!!rc ^ (condition & 1));
786 static void fetch_register_operand(struct operand *op)
790 op->val = *(u8 *)op->addr.reg;
793 op->val = *(u16 *)op->addr.reg;
796 op->val = *(u32 *)op->addr.reg;
799 op->val = *(u64 *)op->addr.reg;
804 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
806 ctxt->ops->get_fpu(ctxt);
808 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
809 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
810 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
811 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
812 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
813 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
814 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
815 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
817 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
818 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
819 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
820 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
821 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
822 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
823 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
824 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
828 ctxt->ops->put_fpu(ctxt);
831 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
834 ctxt->ops->get_fpu(ctxt);
836 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
837 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
838 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
839 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
840 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
841 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
842 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
843 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
845 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
846 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
847 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
848 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
849 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
850 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
851 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
852 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
856 ctxt->ops->put_fpu(ctxt);
859 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
863 unsigned reg = ctxt->modrm_reg;
864 int highbyte_regs = ctxt->rex_prefix == 0;
866 if (!(ctxt->d & ModRM))
867 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
873 read_sse_reg(ctxt, &op->vec_val, reg);
878 if ((ctxt->d & ByteOp) && !inhibit_bytereg) {
879 op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
882 op->addr.reg = decode_register(reg, ctxt->regs, 0);
883 op->bytes = ctxt->op_bytes;
885 fetch_register_operand(op);
886 op->orig_val = op->val;
889 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
893 int index_reg = 0, base_reg = 0, scale;
894 int rc = X86EMUL_CONTINUE;
897 if (ctxt->rex_prefix) {
898 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
899 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
900 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
903 ctxt->modrm = insn_fetch(u8, ctxt);
904 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
905 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
906 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
907 ctxt->modrm_seg = VCPU_SREG_DS;
909 if (ctxt->modrm_mod == 3) {
911 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
912 op->addr.reg = decode_register(ctxt->modrm_rm,
913 ctxt->regs, ctxt->d & ByteOp);
917 op->addr.xmm = ctxt->modrm_rm;
918 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
921 fetch_register_operand(op);
927 if (ctxt->ad_bytes == 2) {
928 unsigned bx = ctxt->regs[VCPU_REGS_RBX];
929 unsigned bp = ctxt->regs[VCPU_REGS_RBP];
930 unsigned si = ctxt->regs[VCPU_REGS_RSI];
931 unsigned di = ctxt->regs[VCPU_REGS_RDI];
933 /* 16-bit ModR/M decode. */
934 switch (ctxt->modrm_mod) {
936 if (ctxt->modrm_rm == 6)
937 modrm_ea += insn_fetch(u16, ctxt);
940 modrm_ea += insn_fetch(s8, ctxt);
943 modrm_ea += insn_fetch(u16, ctxt);
946 switch (ctxt->modrm_rm) {
966 if (ctxt->modrm_mod != 0)
973 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
974 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
975 ctxt->modrm_seg = VCPU_SREG_SS;
976 modrm_ea = (u16)modrm_ea;
978 /* 32/64-bit ModR/M decode. */
979 if ((ctxt->modrm_rm & 7) == 4) {
980 sib = insn_fetch(u8, ctxt);
981 index_reg |= (sib >> 3) & 7;
985 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
986 modrm_ea += insn_fetch(s32, ctxt);
988 modrm_ea += ctxt->regs[base_reg];
990 modrm_ea += ctxt->regs[index_reg] << scale;
991 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
992 if (ctxt->mode == X86EMUL_MODE_PROT64)
993 ctxt->rip_relative = 1;
995 modrm_ea += ctxt->regs[ctxt->modrm_rm];
996 switch (ctxt->modrm_mod) {
998 if (ctxt->modrm_rm == 5)
999 modrm_ea += insn_fetch(s32, ctxt);
1002 modrm_ea += insn_fetch(s8, ctxt);
1005 modrm_ea += insn_fetch(s32, ctxt);
1009 op->addr.mem.ea = modrm_ea;
1014 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1017 int rc = X86EMUL_CONTINUE;
1020 switch (ctxt->ad_bytes) {
1022 op->addr.mem.ea = insn_fetch(u16, ctxt);
1025 op->addr.mem.ea = insn_fetch(u32, ctxt);
1028 op->addr.mem.ea = insn_fetch(u64, ctxt);
1035 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1039 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1040 mask = ~(ctxt->dst.bytes * 8 - 1);
1042 if (ctxt->src.bytes == 2)
1043 sv = (s16)ctxt->src.val & (s16)mask;
1044 else if (ctxt->src.bytes == 4)
1045 sv = (s32)ctxt->src.val & (s32)mask;
1047 ctxt->dst.addr.mem.ea += (sv >> 3);
1050 /* only subword offset */
1051 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1054 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1055 unsigned long addr, void *dest, unsigned size)
1058 struct read_cache *mc = &ctxt->mem_read;
1061 int n = min(size, 8u);
1063 if (mc->pos < mc->end)
1066 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1068 if (rc != X86EMUL_CONTINUE)
1073 memcpy(dest, mc->data + mc->pos, n);
1078 return X86EMUL_CONTINUE;
1081 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1082 struct segmented_address addr,
1089 rc = linearize(ctxt, addr, size, false, &linear);
1090 if (rc != X86EMUL_CONTINUE)
1092 return read_emulated(ctxt, linear, data, size);
1095 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1096 struct segmented_address addr,
1103 rc = linearize(ctxt, addr, size, true, &linear);
1104 if (rc != X86EMUL_CONTINUE)
1106 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1110 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1111 struct segmented_address addr,
1112 const void *orig_data, const void *data,
1118 rc = linearize(ctxt, addr, size, true, &linear);
1119 if (rc != X86EMUL_CONTINUE)
1121 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1122 size, &ctxt->exception);
1125 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1126 unsigned int size, unsigned short port,
1129 struct read_cache *rc = &ctxt->io_read;
1131 if (rc->pos == rc->end) { /* refill pio read ahead */
1132 unsigned int in_page, n;
1133 unsigned int count = ctxt->rep_prefix ?
1134 address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
1135 in_page = (ctxt->eflags & EFLG_DF) ?
1136 offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
1137 PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
1138 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1142 rc->pos = rc->end = 0;
1143 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1148 memcpy(dest, rc->data + rc->pos, size);
1153 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1154 u16 selector, struct desc_ptr *dt)
1156 struct x86_emulate_ops *ops = ctxt->ops;
1158 if (selector & 1 << 2) {
1159 struct desc_struct desc;
1162 memset (dt, 0, sizeof *dt);
1163 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1166 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1167 dt->address = get_desc_base(&desc);
1169 ops->get_gdt(ctxt, dt);
1172 /* allowed just for 8 bytes segments */
1173 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1174 u16 selector, struct desc_struct *desc)
1177 u16 index = selector >> 3;
1180 get_descriptor_table_ptr(ctxt, selector, &dt);
1182 if (dt.size < index * 8 + 7)
1183 return emulate_gp(ctxt, selector & 0xfffc);
1185 addr = dt.address + index * 8;
1186 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1190 /* allowed just for 8 bytes segments */
1191 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1192 u16 selector, struct desc_struct *desc)
1195 u16 index = selector >> 3;
1198 get_descriptor_table_ptr(ctxt, selector, &dt);
1200 if (dt.size < index * 8 + 7)
1201 return emulate_gp(ctxt, selector & 0xfffc);
1203 addr = dt.address + index * 8;
1204 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1208 /* Does not support long mode */
1209 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1210 u16 selector, int seg)
1212 struct desc_struct seg_desc;
1214 unsigned err_vec = GP_VECTOR;
1216 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1219 memset(&seg_desc, 0, sizeof seg_desc);
1221 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1222 || ctxt->mode == X86EMUL_MODE_REAL) {
1223 /* set real mode segment descriptor */
1224 set_desc_base(&seg_desc, selector << 4);
1225 set_desc_limit(&seg_desc, 0xffff);
1232 /* NULL selector is not valid for TR, CS and SS */
1233 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1237 /* TR should be in GDT only */
1238 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1241 if (null_selector) /* for NULL selector skip all following checks */
1244 ret = read_segment_descriptor(ctxt, selector, &seg_desc);
1245 if (ret != X86EMUL_CONTINUE)
1248 err_code = selector & 0xfffc;
1249 err_vec = GP_VECTOR;
1251 /* can't load system descriptor into segment selecor */
1252 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1256 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1262 cpl = ctxt->ops->cpl(ctxt);
1267 * segment is not a writable data segment or segment
1268 * selector's RPL != CPL or segment selector's RPL != CPL
1270 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1274 if (!(seg_desc.type & 8))
1277 if (seg_desc.type & 4) {
1283 if (rpl > cpl || dpl != cpl)
1286 /* CS(RPL) <- CPL */
1287 selector = (selector & 0xfffc) | cpl;
1290 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1293 case VCPU_SREG_LDTR:
1294 if (seg_desc.s || seg_desc.type != 2)
1297 default: /* DS, ES, FS, or GS */
1299 * segment is not a data or readable code segment or
1300 * ((segment is a data or nonconforming code segment)
1301 * and (both RPL and CPL > DPL))
1303 if ((seg_desc.type & 0xa) == 0x8 ||
1304 (((seg_desc.type & 0xc) != 0xc) &&
1305 (rpl > dpl && cpl > dpl)))
1311 /* mark segment as accessed */
1313 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1314 if (ret != X86EMUL_CONTINUE)
1318 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1319 return X86EMUL_CONTINUE;
1321 emulate_exception(ctxt, err_vec, err_code, true);
1322 return X86EMUL_PROPAGATE_FAULT;
1325 static void write_register_operand(struct operand *op)
1327 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1328 switch (op->bytes) {
1330 *(u8 *)op->addr.reg = (u8)op->val;
1333 *(u16 *)op->addr.reg = (u16)op->val;
1336 *op->addr.reg = (u32)op->val;
1337 break; /* 64b: zero-extend */
1339 *op->addr.reg = op->val;
1344 static int writeback(struct x86_emulate_ctxt *ctxt)
1348 switch (ctxt->dst.type) {
1350 write_register_operand(&ctxt->dst);
1353 if (ctxt->lock_prefix)
1354 rc = segmented_cmpxchg(ctxt,
1356 &ctxt->dst.orig_val,
1360 rc = segmented_write(ctxt,
1364 if (rc != X86EMUL_CONTINUE)
1368 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1376 return X86EMUL_CONTINUE;
1379 static int em_push(struct x86_emulate_ctxt *ctxt)
1381 struct segmented_address addr;
1383 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], -ctxt->op_bytes);
1384 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1385 addr.seg = VCPU_SREG_SS;
1387 /* Disable writeback. */
1388 ctxt->dst.type = OP_NONE;
1389 return segmented_write(ctxt, addr, &ctxt->src.val, ctxt->op_bytes);
1392 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1393 void *dest, int len)
1396 struct segmented_address addr;
1398 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1399 addr.seg = VCPU_SREG_SS;
1400 rc = segmented_read(ctxt, addr, dest, len);
1401 if (rc != X86EMUL_CONTINUE)
1404 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], len);
1408 static int em_pop(struct x86_emulate_ctxt *ctxt)
1410 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1413 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1414 void *dest, int len)
1417 unsigned long val, change_mask;
1418 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1419 int cpl = ctxt->ops->cpl(ctxt);
1421 rc = emulate_pop(ctxt, &val, len);
1422 if (rc != X86EMUL_CONTINUE)
1425 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1426 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1428 switch(ctxt->mode) {
1429 case X86EMUL_MODE_PROT64:
1430 case X86EMUL_MODE_PROT32:
1431 case X86EMUL_MODE_PROT16:
1433 change_mask |= EFLG_IOPL;
1435 change_mask |= EFLG_IF;
1437 case X86EMUL_MODE_VM86:
1439 return emulate_gp(ctxt, 0);
1440 change_mask |= EFLG_IF;
1442 default: /* real mode */
1443 change_mask |= (EFLG_IOPL | EFLG_IF);
1447 *(unsigned long *)dest =
1448 (ctxt->eflags & ~change_mask) | (val & change_mask);
1453 static int em_popf(struct x86_emulate_ctxt *ctxt)
1455 ctxt->dst.type = OP_REG;
1456 ctxt->dst.addr.reg = &ctxt->eflags;
1457 ctxt->dst.bytes = ctxt->op_bytes;
1458 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1461 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1463 int seg = ctxt->src2.val;
1465 ctxt->src.val = get_segment_selector(ctxt, seg);
1467 return em_push(ctxt);
1470 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1472 int seg = ctxt->src2.val;
1473 unsigned long selector;
1476 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1477 if (rc != X86EMUL_CONTINUE)
1480 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1484 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1486 unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
1487 int rc = X86EMUL_CONTINUE;
1488 int reg = VCPU_REGS_RAX;
1490 while (reg <= VCPU_REGS_RDI) {
1491 (reg == VCPU_REGS_RSP) ?
1492 (ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
1495 if (rc != X86EMUL_CONTINUE)
1504 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1506 ctxt->src.val = (unsigned long)ctxt->eflags;
1507 return em_push(ctxt);
1510 static int em_popa(struct x86_emulate_ctxt *ctxt)
1512 int rc = X86EMUL_CONTINUE;
1513 int reg = VCPU_REGS_RDI;
1515 while (reg >= VCPU_REGS_RAX) {
1516 if (reg == VCPU_REGS_RSP) {
1517 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP],
1522 rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
1523 if (rc != X86EMUL_CONTINUE)
1530 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1532 struct x86_emulate_ops *ops = ctxt->ops;
1539 /* TODO: Add limit checks */
1540 ctxt->src.val = ctxt->eflags;
1542 if (rc != X86EMUL_CONTINUE)
1545 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1547 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1549 if (rc != X86EMUL_CONTINUE)
1552 ctxt->src.val = ctxt->_eip;
1554 if (rc != X86EMUL_CONTINUE)
1557 ops->get_idt(ctxt, &dt);
1559 eip_addr = dt.address + (irq << 2);
1560 cs_addr = dt.address + (irq << 2) + 2;
1562 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1563 if (rc != X86EMUL_CONTINUE)
1566 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1567 if (rc != X86EMUL_CONTINUE)
1570 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1571 if (rc != X86EMUL_CONTINUE)
1579 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1581 switch(ctxt->mode) {
1582 case X86EMUL_MODE_REAL:
1583 return emulate_int_real(ctxt, irq);
1584 case X86EMUL_MODE_VM86:
1585 case X86EMUL_MODE_PROT16:
1586 case X86EMUL_MODE_PROT32:
1587 case X86EMUL_MODE_PROT64:
1589 /* Protected mode interrupts unimplemented yet */
1590 return X86EMUL_UNHANDLEABLE;
1594 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1596 int rc = X86EMUL_CONTINUE;
1597 unsigned long temp_eip = 0;
1598 unsigned long temp_eflags = 0;
1599 unsigned long cs = 0;
1600 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1601 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1602 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1603 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1605 /* TODO: Add stack limit check */
1607 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1609 if (rc != X86EMUL_CONTINUE)
1612 if (temp_eip & ~0xffff)
1613 return emulate_gp(ctxt, 0);
1615 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1617 if (rc != X86EMUL_CONTINUE)
1620 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1622 if (rc != X86EMUL_CONTINUE)
1625 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1627 if (rc != X86EMUL_CONTINUE)
1630 ctxt->_eip = temp_eip;
1633 if (ctxt->op_bytes == 4)
1634 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1635 else if (ctxt->op_bytes == 2) {
1636 ctxt->eflags &= ~0xffff;
1637 ctxt->eflags |= temp_eflags;
1640 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1641 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1646 static int em_iret(struct x86_emulate_ctxt *ctxt)
1648 switch(ctxt->mode) {
1649 case X86EMUL_MODE_REAL:
1650 return emulate_iret_real(ctxt);
1651 case X86EMUL_MODE_VM86:
1652 case X86EMUL_MODE_PROT16:
1653 case X86EMUL_MODE_PROT32:
1654 case X86EMUL_MODE_PROT64:
1656 /* iret from protected mode unimplemented yet */
1657 return X86EMUL_UNHANDLEABLE;
1661 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1666 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1668 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1669 if (rc != X86EMUL_CONTINUE)
1673 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1674 return X86EMUL_CONTINUE;
1677 static int em_grp1a(struct x86_emulate_ctxt *ctxt)
1679 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->dst.bytes);
1682 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1684 switch (ctxt->modrm_reg) {
1686 emulate_2op_SrcB(ctxt, "rol");
1689 emulate_2op_SrcB(ctxt, "ror");
1692 emulate_2op_SrcB(ctxt, "rcl");
1695 emulate_2op_SrcB(ctxt, "rcr");
1697 case 4: /* sal/shl */
1698 case 6: /* sal/shl */
1699 emulate_2op_SrcB(ctxt, "sal");
1702 emulate_2op_SrcB(ctxt, "shr");
1705 emulate_2op_SrcB(ctxt, "sar");
1708 return X86EMUL_CONTINUE;
1711 static int em_not(struct x86_emulate_ctxt *ctxt)
1713 ctxt->dst.val = ~ctxt->dst.val;
1714 return X86EMUL_CONTINUE;
1717 static int em_neg(struct x86_emulate_ctxt *ctxt)
1719 emulate_1op(ctxt, "neg");
1720 return X86EMUL_CONTINUE;
1723 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
1727 emulate_1op_rax_rdx(ctxt, "mul", ex);
1728 return X86EMUL_CONTINUE;
1731 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
1735 emulate_1op_rax_rdx(ctxt, "imul", ex);
1736 return X86EMUL_CONTINUE;
1739 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1743 emulate_1op_rax_rdx(ctxt, "div", de);
1745 return emulate_de(ctxt);
1746 return X86EMUL_CONTINUE;
1749 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
1753 emulate_1op_rax_rdx(ctxt, "idiv", de);
1755 return emulate_de(ctxt);
1756 return X86EMUL_CONTINUE;
1759 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1761 int rc = X86EMUL_CONTINUE;
1763 switch (ctxt->modrm_reg) {
1765 emulate_1op(ctxt, "inc");
1768 emulate_1op(ctxt, "dec");
1770 case 2: /* call near abs */ {
1772 old_eip = ctxt->_eip;
1773 ctxt->_eip = ctxt->src.val;
1774 ctxt->src.val = old_eip;
1778 case 4: /* jmp abs */
1779 ctxt->_eip = ctxt->src.val;
1781 case 5: /* jmp far */
1782 rc = em_jmp_far(ctxt);
1791 static int em_grp9(struct x86_emulate_ctxt *ctxt)
1793 u64 old = ctxt->dst.orig_val64;
1795 if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
1796 ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
1797 ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1798 ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1799 ctxt->eflags &= ~EFLG_ZF;
1801 ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
1802 (u32) ctxt->regs[VCPU_REGS_RBX];
1804 ctxt->eflags |= EFLG_ZF;
1806 return X86EMUL_CONTINUE;
1809 static int em_ret(struct x86_emulate_ctxt *ctxt)
1811 ctxt->dst.type = OP_REG;
1812 ctxt->dst.addr.reg = &ctxt->_eip;
1813 ctxt->dst.bytes = ctxt->op_bytes;
1814 return em_pop(ctxt);
1817 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
1822 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
1823 if (rc != X86EMUL_CONTINUE)
1825 if (ctxt->op_bytes == 4)
1826 ctxt->_eip = (u32)ctxt->_eip;
1827 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1828 if (rc != X86EMUL_CONTINUE)
1830 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1834 static int em_lseg(struct x86_emulate_ctxt *ctxt)
1836 int seg = ctxt->src2.val;
1840 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1842 rc = load_segment_descriptor(ctxt, sel, seg);
1843 if (rc != X86EMUL_CONTINUE)
1846 ctxt->dst.val = ctxt->src.val;
1851 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1852 struct desc_struct *cs, struct desc_struct *ss)
1856 memset(cs, 0, sizeof(struct desc_struct));
1857 ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1858 memset(ss, 0, sizeof(struct desc_struct));
1860 cs->l = 0; /* will be adjusted later */
1861 set_desc_base(cs, 0); /* flat segment */
1862 cs->g = 1; /* 4kb granularity */
1863 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1864 cs->type = 0x0b; /* Read, Execute, Accessed */
1866 cs->dpl = 0; /* will be adjusted later */
1870 set_desc_base(ss, 0); /* flat segment */
1871 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1872 ss->g = 1; /* 4kb granularity */
1874 ss->type = 0x03; /* Read/Write, Accessed */
1875 ss->d = 1; /* 32bit stack segment */
1880 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
1882 struct x86_emulate_ops *ops = ctxt->ops;
1883 u32 eax, ebx, ecx, edx;
1886 * syscall should always be enabled in longmode - so only become
1887 * vendor specific (cpuid) if other modes are active...
1889 if (ctxt->mode == X86EMUL_MODE_PROT64)
1894 if (ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx)) {
1896 * Intel ("GenuineIntel")
1897 * remark: Intel CPUs only support "syscall" in 64bit
1898 * longmode. Also an 64bit guest with a
1899 * 32bit compat-app running will #UD !! While this
1900 * behaviour can be fixed (by emulating) into AMD
1901 * response - CPUs of AMD can't behave like Intel.
1903 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
1904 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
1905 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
1908 /* AMD ("AuthenticAMD") */
1909 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
1910 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
1911 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
1914 /* AMD ("AMDisbetter!") */
1915 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
1916 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
1917 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
1921 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
1925 static int em_syscall(struct x86_emulate_ctxt *ctxt)
1927 struct x86_emulate_ops *ops = ctxt->ops;
1928 struct desc_struct cs, ss;
1933 /* syscall is not available in real mode */
1934 if (ctxt->mode == X86EMUL_MODE_REAL ||
1935 ctxt->mode == X86EMUL_MODE_VM86)
1936 return emulate_ud(ctxt);
1938 if (!(em_syscall_is_enabled(ctxt)))
1939 return emulate_ud(ctxt);
1941 ops->get_msr(ctxt, MSR_EFER, &efer);
1942 setup_syscalls_segments(ctxt, &cs, &ss);
1944 if (!(efer & EFER_SCE))
1945 return emulate_ud(ctxt);
1947 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1949 cs_sel = (u16)(msr_data & 0xfffc);
1950 ss_sel = (u16)(msr_data + 8);
1952 if (efer & EFER_LMA) {
1956 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1957 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1959 ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
1960 if (efer & EFER_LMA) {
1961 #ifdef CONFIG_X86_64
1962 ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1965 ctxt->mode == X86EMUL_MODE_PROT64 ?
1966 MSR_LSTAR : MSR_CSTAR, &msr_data);
1967 ctxt->_eip = msr_data;
1969 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1970 ctxt->eflags &= ~(msr_data | EFLG_RF);
1974 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1975 ctxt->_eip = (u32)msr_data;
1977 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1980 return X86EMUL_CONTINUE;
1983 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
1985 struct x86_emulate_ops *ops = ctxt->ops;
1986 struct desc_struct cs, ss;
1991 ops->get_msr(ctxt, MSR_EFER, &efer);
1992 /* inject #GP if in real mode */
1993 if (ctxt->mode == X86EMUL_MODE_REAL)
1994 return emulate_gp(ctxt, 0);
1996 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1997 * Therefore, we inject an #UD.
1999 if (ctxt->mode == X86EMUL_MODE_PROT64)
2000 return emulate_ud(ctxt);
2002 setup_syscalls_segments(ctxt, &cs, &ss);
2004 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2005 switch (ctxt->mode) {
2006 case X86EMUL_MODE_PROT32:
2007 if ((msr_data & 0xfffc) == 0x0)
2008 return emulate_gp(ctxt, 0);
2010 case X86EMUL_MODE_PROT64:
2011 if (msr_data == 0x0)
2012 return emulate_gp(ctxt, 0);
2016 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2017 cs_sel = (u16)msr_data;
2018 cs_sel &= ~SELECTOR_RPL_MASK;
2019 ss_sel = cs_sel + 8;
2020 ss_sel &= ~SELECTOR_RPL_MASK;
2021 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2026 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2027 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2029 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2030 ctxt->_eip = msr_data;
2032 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2033 ctxt->regs[VCPU_REGS_RSP] = msr_data;
2035 return X86EMUL_CONTINUE;
2038 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2040 struct x86_emulate_ops *ops = ctxt->ops;
2041 struct desc_struct cs, ss;
2044 u16 cs_sel = 0, ss_sel = 0;
2046 /* inject #GP if in real mode or Virtual 8086 mode */
2047 if (ctxt->mode == X86EMUL_MODE_REAL ||
2048 ctxt->mode == X86EMUL_MODE_VM86)
2049 return emulate_gp(ctxt, 0);
2051 setup_syscalls_segments(ctxt, &cs, &ss);
2053 if ((ctxt->rex_prefix & 0x8) != 0x0)
2054 usermode = X86EMUL_MODE_PROT64;
2056 usermode = X86EMUL_MODE_PROT32;
2060 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2062 case X86EMUL_MODE_PROT32:
2063 cs_sel = (u16)(msr_data + 16);
2064 if ((msr_data & 0xfffc) == 0x0)
2065 return emulate_gp(ctxt, 0);
2066 ss_sel = (u16)(msr_data + 24);
2068 case X86EMUL_MODE_PROT64:
2069 cs_sel = (u16)(msr_data + 32);
2070 if (msr_data == 0x0)
2071 return emulate_gp(ctxt, 0);
2072 ss_sel = cs_sel + 8;
2077 cs_sel |= SELECTOR_RPL_MASK;
2078 ss_sel |= SELECTOR_RPL_MASK;
2080 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2081 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2083 ctxt->_eip = ctxt->regs[VCPU_REGS_RDX];
2084 ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX];
2086 return X86EMUL_CONTINUE;
2089 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2092 if (ctxt->mode == X86EMUL_MODE_REAL)
2094 if (ctxt->mode == X86EMUL_MODE_VM86)
2096 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2097 return ctxt->ops->cpl(ctxt) > iopl;
2100 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2103 struct x86_emulate_ops *ops = ctxt->ops;
2104 struct desc_struct tr_seg;
2107 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2108 unsigned mask = (1 << len) - 1;
2111 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2114 if (desc_limit_scaled(&tr_seg) < 103)
2116 base = get_desc_base(&tr_seg);
2117 #ifdef CONFIG_X86_64
2118 base |= ((u64)base3) << 32;
2120 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2121 if (r != X86EMUL_CONTINUE)
2123 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2125 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2126 if (r != X86EMUL_CONTINUE)
2128 if ((perm >> bit_idx) & mask)
2133 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2139 if (emulator_bad_iopl(ctxt))
2140 if (!emulator_io_port_access_allowed(ctxt, port, len))
2143 ctxt->perm_ok = true;
2148 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2149 struct tss_segment_16 *tss)
2151 tss->ip = ctxt->_eip;
2152 tss->flag = ctxt->eflags;
2153 tss->ax = ctxt->regs[VCPU_REGS_RAX];
2154 tss->cx = ctxt->regs[VCPU_REGS_RCX];
2155 tss->dx = ctxt->regs[VCPU_REGS_RDX];
2156 tss->bx = ctxt->regs[VCPU_REGS_RBX];
2157 tss->sp = ctxt->regs[VCPU_REGS_RSP];
2158 tss->bp = ctxt->regs[VCPU_REGS_RBP];
2159 tss->si = ctxt->regs[VCPU_REGS_RSI];
2160 tss->di = ctxt->regs[VCPU_REGS_RDI];
2162 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2163 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2164 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2165 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2166 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2169 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2170 struct tss_segment_16 *tss)
2174 ctxt->_eip = tss->ip;
2175 ctxt->eflags = tss->flag | 2;
2176 ctxt->regs[VCPU_REGS_RAX] = tss->ax;
2177 ctxt->regs[VCPU_REGS_RCX] = tss->cx;
2178 ctxt->regs[VCPU_REGS_RDX] = tss->dx;
2179 ctxt->regs[VCPU_REGS_RBX] = tss->bx;
2180 ctxt->regs[VCPU_REGS_RSP] = tss->sp;
2181 ctxt->regs[VCPU_REGS_RBP] = tss->bp;
2182 ctxt->regs[VCPU_REGS_RSI] = tss->si;
2183 ctxt->regs[VCPU_REGS_RDI] = tss->di;
2186 * SDM says that segment selectors are loaded before segment
2189 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2190 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2191 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2192 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2193 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2196 * Now load segment descriptors. If fault happenes at this stage
2197 * it is handled in a context of new task
2199 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2200 if (ret != X86EMUL_CONTINUE)
2202 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2203 if (ret != X86EMUL_CONTINUE)
2205 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2206 if (ret != X86EMUL_CONTINUE)
2208 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2209 if (ret != X86EMUL_CONTINUE)
2211 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2212 if (ret != X86EMUL_CONTINUE)
2215 return X86EMUL_CONTINUE;
2218 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2219 u16 tss_selector, u16 old_tss_sel,
2220 ulong old_tss_base, struct desc_struct *new_desc)
2222 struct x86_emulate_ops *ops = ctxt->ops;
2223 struct tss_segment_16 tss_seg;
2225 u32 new_tss_base = get_desc_base(new_desc);
2227 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2229 if (ret != X86EMUL_CONTINUE)
2230 /* FIXME: need to provide precise fault address */
2233 save_state_to_tss16(ctxt, &tss_seg);
2235 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2237 if (ret != X86EMUL_CONTINUE)
2238 /* FIXME: need to provide precise fault address */
2241 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2243 if (ret != X86EMUL_CONTINUE)
2244 /* FIXME: need to provide precise fault address */
2247 if (old_tss_sel != 0xffff) {
2248 tss_seg.prev_task_link = old_tss_sel;
2250 ret = ops->write_std(ctxt, new_tss_base,
2251 &tss_seg.prev_task_link,
2252 sizeof tss_seg.prev_task_link,
2254 if (ret != X86EMUL_CONTINUE)
2255 /* FIXME: need to provide precise fault address */
2259 return load_state_from_tss16(ctxt, &tss_seg);
2262 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2263 struct tss_segment_32 *tss)
2265 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2266 tss->eip = ctxt->_eip;
2267 tss->eflags = ctxt->eflags;
2268 tss->eax = ctxt->regs[VCPU_REGS_RAX];
2269 tss->ecx = ctxt->regs[VCPU_REGS_RCX];
2270 tss->edx = ctxt->regs[VCPU_REGS_RDX];
2271 tss->ebx = ctxt->regs[VCPU_REGS_RBX];
2272 tss->esp = ctxt->regs[VCPU_REGS_RSP];
2273 tss->ebp = ctxt->regs[VCPU_REGS_RBP];
2274 tss->esi = ctxt->regs[VCPU_REGS_RSI];
2275 tss->edi = ctxt->regs[VCPU_REGS_RDI];
2277 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2278 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2279 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2280 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2281 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2282 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2283 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2286 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2287 struct tss_segment_32 *tss)
2291 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2292 return emulate_gp(ctxt, 0);
2293 ctxt->_eip = tss->eip;
2294 ctxt->eflags = tss->eflags | 2;
2295 ctxt->regs[VCPU_REGS_RAX] = tss->eax;
2296 ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
2297 ctxt->regs[VCPU_REGS_RDX] = tss->edx;
2298 ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
2299 ctxt->regs[VCPU_REGS_RSP] = tss->esp;
2300 ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
2301 ctxt->regs[VCPU_REGS_RSI] = tss->esi;
2302 ctxt->regs[VCPU_REGS_RDI] = tss->edi;
2305 * SDM says that segment selectors are loaded before segment
2308 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2309 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2310 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2311 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2312 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2313 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2314 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2317 * Now load segment descriptors. If fault happenes at this stage
2318 * it is handled in a context of new task
2320 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2321 if (ret != X86EMUL_CONTINUE)
2323 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2324 if (ret != X86EMUL_CONTINUE)
2326 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2327 if (ret != X86EMUL_CONTINUE)
2329 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2330 if (ret != X86EMUL_CONTINUE)
2332 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2333 if (ret != X86EMUL_CONTINUE)
2335 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2336 if (ret != X86EMUL_CONTINUE)
2338 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2339 if (ret != X86EMUL_CONTINUE)
2342 return X86EMUL_CONTINUE;
2345 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2346 u16 tss_selector, u16 old_tss_sel,
2347 ulong old_tss_base, struct desc_struct *new_desc)
2349 struct x86_emulate_ops *ops = ctxt->ops;
2350 struct tss_segment_32 tss_seg;
2352 u32 new_tss_base = get_desc_base(new_desc);
2354 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2356 if (ret != X86EMUL_CONTINUE)
2357 /* FIXME: need to provide precise fault address */
2360 save_state_to_tss32(ctxt, &tss_seg);
2362 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2364 if (ret != X86EMUL_CONTINUE)
2365 /* FIXME: need to provide precise fault address */
2368 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2370 if (ret != X86EMUL_CONTINUE)
2371 /* FIXME: need to provide precise fault address */
2374 if (old_tss_sel != 0xffff) {
2375 tss_seg.prev_task_link = old_tss_sel;
2377 ret = ops->write_std(ctxt, new_tss_base,
2378 &tss_seg.prev_task_link,
2379 sizeof tss_seg.prev_task_link,
2381 if (ret != X86EMUL_CONTINUE)
2382 /* FIXME: need to provide precise fault address */
2386 return load_state_from_tss32(ctxt, &tss_seg);
2389 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2390 u16 tss_selector, int reason,
2391 bool has_error_code, u32 error_code)
2393 struct x86_emulate_ops *ops = ctxt->ops;
2394 struct desc_struct curr_tss_desc, next_tss_desc;
2396 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2397 ulong old_tss_base =
2398 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2401 /* FIXME: old_tss_base == ~0 ? */
2403 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2404 if (ret != X86EMUL_CONTINUE)
2406 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2407 if (ret != X86EMUL_CONTINUE)
2410 /* FIXME: check that next_tss_desc is tss */
2412 if (reason != TASK_SWITCH_IRET) {
2413 if ((tss_selector & 3) > next_tss_desc.dpl ||
2414 ops->cpl(ctxt) > next_tss_desc.dpl)
2415 return emulate_gp(ctxt, 0);
2418 desc_limit = desc_limit_scaled(&next_tss_desc);
2419 if (!next_tss_desc.p ||
2420 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2421 desc_limit < 0x2b)) {
2422 emulate_ts(ctxt, tss_selector & 0xfffc);
2423 return X86EMUL_PROPAGATE_FAULT;
2426 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2427 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2428 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2431 if (reason == TASK_SWITCH_IRET)
2432 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2434 /* set back link to prev task only if NT bit is set in eflags
2435 note that old_tss_sel is not used afetr this point */
2436 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2437 old_tss_sel = 0xffff;
2439 if (next_tss_desc.type & 8)
2440 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2441 old_tss_base, &next_tss_desc);
2443 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2444 old_tss_base, &next_tss_desc);
2445 if (ret != X86EMUL_CONTINUE)
2448 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2449 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2451 if (reason != TASK_SWITCH_IRET) {
2452 next_tss_desc.type |= (1 << 1); /* set busy flag */
2453 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2456 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2457 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2459 if (has_error_code) {
2460 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2461 ctxt->lock_prefix = 0;
2462 ctxt->src.val = (unsigned long) error_code;
2463 ret = em_push(ctxt);
2469 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2470 u16 tss_selector, int reason,
2471 bool has_error_code, u32 error_code)
2475 ctxt->_eip = ctxt->eip;
2476 ctxt->dst.type = OP_NONE;
2478 rc = emulator_do_task_switch(ctxt, tss_selector, reason,
2479 has_error_code, error_code);
2481 if (rc == X86EMUL_CONTINUE)
2482 ctxt->eip = ctxt->_eip;
2484 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2487 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2488 int reg, struct operand *op)
2490 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2492 register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
2493 op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
2494 op->addr.mem.seg = seg;
2497 static int em_das(struct x86_emulate_ctxt *ctxt)
2500 bool af, cf, old_cf;
2502 cf = ctxt->eflags & X86_EFLAGS_CF;
2508 af = ctxt->eflags & X86_EFLAGS_AF;
2509 if ((al & 0x0f) > 9 || af) {
2511 cf = old_cf | (al >= 250);
2516 if (old_al > 0x99 || old_cf) {
2522 /* Set PF, ZF, SF */
2523 ctxt->src.type = OP_IMM;
2525 ctxt->src.bytes = 1;
2526 emulate_2op_SrcV(ctxt, "or");
2527 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2529 ctxt->eflags |= X86_EFLAGS_CF;
2531 ctxt->eflags |= X86_EFLAGS_AF;
2532 return X86EMUL_CONTINUE;
2535 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2541 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2542 old_eip = ctxt->_eip;
2544 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2545 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2546 return X86EMUL_CONTINUE;
2549 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2551 ctxt->src.val = old_cs;
2553 if (rc != X86EMUL_CONTINUE)
2556 ctxt->src.val = old_eip;
2557 return em_push(ctxt);
2560 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2564 ctxt->dst.type = OP_REG;
2565 ctxt->dst.addr.reg = &ctxt->_eip;
2566 ctxt->dst.bytes = ctxt->op_bytes;
2567 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2568 if (rc != X86EMUL_CONTINUE)
2570 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], ctxt->src.val);
2571 return X86EMUL_CONTINUE;
2574 static int em_add(struct x86_emulate_ctxt *ctxt)
2576 emulate_2op_SrcV(ctxt, "add");
2577 return X86EMUL_CONTINUE;
2580 static int em_or(struct x86_emulate_ctxt *ctxt)
2582 emulate_2op_SrcV(ctxt, "or");
2583 return X86EMUL_CONTINUE;
2586 static int em_adc(struct x86_emulate_ctxt *ctxt)
2588 emulate_2op_SrcV(ctxt, "adc");
2589 return X86EMUL_CONTINUE;
2592 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2594 emulate_2op_SrcV(ctxt, "sbb");
2595 return X86EMUL_CONTINUE;
2598 static int em_and(struct x86_emulate_ctxt *ctxt)
2600 emulate_2op_SrcV(ctxt, "and");
2601 return X86EMUL_CONTINUE;
2604 static int em_sub(struct x86_emulate_ctxt *ctxt)
2606 emulate_2op_SrcV(ctxt, "sub");
2607 return X86EMUL_CONTINUE;
2610 static int em_xor(struct x86_emulate_ctxt *ctxt)
2612 emulate_2op_SrcV(ctxt, "xor");
2613 return X86EMUL_CONTINUE;
2616 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2618 emulate_2op_SrcV(ctxt, "cmp");
2619 /* Disable writeback. */
2620 ctxt->dst.type = OP_NONE;
2621 return X86EMUL_CONTINUE;
2624 static int em_test(struct x86_emulate_ctxt *ctxt)
2626 emulate_2op_SrcV(ctxt, "test");
2627 /* Disable writeback. */
2628 ctxt->dst.type = OP_NONE;
2629 return X86EMUL_CONTINUE;
2632 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2634 /* Write back the register source. */
2635 ctxt->src.val = ctxt->dst.val;
2636 write_register_operand(&ctxt->src);
2638 /* Write back the memory destination with implicit LOCK prefix. */
2639 ctxt->dst.val = ctxt->src.orig_val;
2640 ctxt->lock_prefix = 1;
2641 return X86EMUL_CONTINUE;
2644 static int em_imul(struct x86_emulate_ctxt *ctxt)
2646 emulate_2op_SrcV_nobyte(ctxt, "imul");
2647 return X86EMUL_CONTINUE;
2650 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2652 ctxt->dst.val = ctxt->src2.val;
2653 return em_imul(ctxt);
2656 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2658 ctxt->dst.type = OP_REG;
2659 ctxt->dst.bytes = ctxt->src.bytes;
2660 ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
2661 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2663 return X86EMUL_CONTINUE;
2666 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2670 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2671 ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
2672 ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
2673 return X86EMUL_CONTINUE;
2676 static int em_mov(struct x86_emulate_ctxt *ctxt)
2678 ctxt->dst.val = ctxt->src.val;
2679 return X86EMUL_CONTINUE;
2682 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
2684 if (ctxt->modrm_reg > VCPU_SREG_GS)
2685 return emulate_ud(ctxt);
2687 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
2688 return X86EMUL_CONTINUE;
2691 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
2693 u16 sel = ctxt->src.val;
2695 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
2696 return emulate_ud(ctxt);
2698 if (ctxt->modrm_reg == VCPU_SREG_SS)
2699 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2701 /* Disable writeback. */
2702 ctxt->dst.type = OP_NONE;
2703 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
2706 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2708 memcpy(&ctxt->dst.vec_val, &ctxt->src.vec_val, ctxt->op_bytes);
2709 return X86EMUL_CONTINUE;
2712 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2717 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
2718 if (rc == X86EMUL_CONTINUE)
2719 ctxt->ops->invlpg(ctxt, linear);
2720 /* Disable writeback. */
2721 ctxt->dst.type = OP_NONE;
2722 return X86EMUL_CONTINUE;
2725 static int em_clts(struct x86_emulate_ctxt *ctxt)
2729 cr0 = ctxt->ops->get_cr(ctxt, 0);
2731 ctxt->ops->set_cr(ctxt, 0, cr0);
2732 return X86EMUL_CONTINUE;
2735 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
2739 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
2740 return X86EMUL_UNHANDLEABLE;
2742 rc = ctxt->ops->fix_hypercall(ctxt);
2743 if (rc != X86EMUL_CONTINUE)
2746 /* Let the processor re-execute the fixed hypercall */
2747 ctxt->_eip = ctxt->eip;
2748 /* Disable writeback. */
2749 ctxt->dst.type = OP_NONE;
2750 return X86EMUL_CONTINUE;
2753 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
2755 struct desc_ptr desc_ptr;
2758 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2759 &desc_ptr.size, &desc_ptr.address,
2761 if (rc != X86EMUL_CONTINUE)
2763 ctxt->ops->set_gdt(ctxt, &desc_ptr);
2764 /* Disable writeback. */
2765 ctxt->dst.type = OP_NONE;
2766 return X86EMUL_CONTINUE;
2769 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2773 rc = ctxt->ops->fix_hypercall(ctxt);
2775 /* Disable writeback. */
2776 ctxt->dst.type = OP_NONE;
2780 static int em_lidt(struct x86_emulate_ctxt *ctxt)
2782 struct desc_ptr desc_ptr;
2785 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2786 &desc_ptr.size, &desc_ptr.address,
2788 if (rc != X86EMUL_CONTINUE)
2790 ctxt->ops->set_idt(ctxt, &desc_ptr);
2791 /* Disable writeback. */
2792 ctxt->dst.type = OP_NONE;
2793 return X86EMUL_CONTINUE;
2796 static int em_smsw(struct x86_emulate_ctxt *ctxt)
2798 ctxt->dst.bytes = 2;
2799 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
2800 return X86EMUL_CONTINUE;
2803 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
2805 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2806 | (ctxt->src.val & 0x0f));
2807 ctxt->dst.type = OP_NONE;
2808 return X86EMUL_CONTINUE;
2811 static int em_loop(struct x86_emulate_ctxt *ctxt)
2813 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
2814 if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
2815 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
2816 jmp_rel(ctxt, ctxt->src.val);
2818 return X86EMUL_CONTINUE;
2821 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
2823 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
2824 jmp_rel(ctxt, ctxt->src.val);
2826 return X86EMUL_CONTINUE;
2829 static int em_cli(struct x86_emulate_ctxt *ctxt)
2831 if (emulator_bad_iopl(ctxt))
2832 return emulate_gp(ctxt, 0);
2834 ctxt->eflags &= ~X86_EFLAGS_IF;
2835 return X86EMUL_CONTINUE;
2838 static int em_sti(struct x86_emulate_ctxt *ctxt)
2840 if (emulator_bad_iopl(ctxt))
2841 return emulate_gp(ctxt, 0);
2843 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
2844 ctxt->eflags |= X86_EFLAGS_IF;
2845 return X86EMUL_CONTINUE;
2848 static bool valid_cr(int nr)
2860 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2862 if (!valid_cr(ctxt->modrm_reg))
2863 return emulate_ud(ctxt);
2865 return X86EMUL_CONTINUE;
2868 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2870 u64 new_val = ctxt->src.val64;
2871 int cr = ctxt->modrm_reg;
2874 static u64 cr_reserved_bits[] = {
2875 0xffffffff00000000ULL,
2876 0, 0, 0, /* CR3 checked later */
2883 return emulate_ud(ctxt);
2885 if (new_val & cr_reserved_bits[cr])
2886 return emulate_gp(ctxt, 0);
2891 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2892 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2893 return emulate_gp(ctxt, 0);
2895 cr4 = ctxt->ops->get_cr(ctxt, 4);
2896 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2898 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2899 !(cr4 & X86_CR4_PAE))
2900 return emulate_gp(ctxt, 0);
2907 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2908 if (efer & EFER_LMA)
2909 rsvd = CR3_L_MODE_RESERVED_BITS;
2910 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
2911 rsvd = CR3_PAE_RESERVED_BITS;
2912 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
2913 rsvd = CR3_NONPAE_RESERVED_BITS;
2916 return emulate_gp(ctxt, 0);
2923 cr4 = ctxt->ops->get_cr(ctxt, 4);
2924 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2926 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2927 return emulate_gp(ctxt, 0);
2933 return X86EMUL_CONTINUE;
2936 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2940 ctxt->ops->get_dr(ctxt, 7, &dr7);
2942 /* Check if DR7.Global_Enable is set */
2943 return dr7 & (1 << 13);
2946 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2948 int dr = ctxt->modrm_reg;
2952 return emulate_ud(ctxt);
2954 cr4 = ctxt->ops->get_cr(ctxt, 4);
2955 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2956 return emulate_ud(ctxt);
2958 if (check_dr7_gd(ctxt))
2959 return emulate_db(ctxt);
2961 return X86EMUL_CONTINUE;
2964 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2966 u64 new_val = ctxt->src.val64;
2967 int dr = ctxt->modrm_reg;
2969 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2970 return emulate_gp(ctxt, 0);
2972 return check_dr_read(ctxt);
2975 static int check_svme(struct x86_emulate_ctxt *ctxt)
2979 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2981 if (!(efer & EFER_SVME))
2982 return emulate_ud(ctxt);
2984 return X86EMUL_CONTINUE;
2987 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
2989 u64 rax = ctxt->regs[VCPU_REGS_RAX];
2991 /* Valid physical address? */
2992 if (rax & 0xffff000000000000ULL)
2993 return emulate_gp(ctxt, 0);
2995 return check_svme(ctxt);
2998 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3000 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3002 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3003 return emulate_ud(ctxt);
3005 return X86EMUL_CONTINUE;
3008 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3010 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3011 u64 rcx = ctxt->regs[VCPU_REGS_RCX];
3013 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3015 return emulate_gp(ctxt, 0);
3017 return X86EMUL_CONTINUE;
3020 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3022 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3023 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3024 return emulate_gp(ctxt, 0);
3026 return X86EMUL_CONTINUE;
3029 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3031 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3032 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3033 return emulate_gp(ctxt, 0);
3035 return X86EMUL_CONTINUE;
3038 #define D(_y) { .flags = (_y) }
3039 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3040 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3041 .check_perm = (_p) }
3043 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3044 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
3045 #define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
3046 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3047 #define II(_f, _e, _i) \
3048 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3049 #define IIP(_f, _e, _i, _p) \
3050 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3051 .check_perm = (_p) }
3052 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3054 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3055 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3056 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3058 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3059 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3060 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3062 static struct opcode group7_rm1[] = {
3063 DI(SrcNone | ModRM | Priv, monitor),
3064 DI(SrcNone | ModRM | Priv, mwait),
3068 static struct opcode group7_rm3[] = {
3069 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
3070 II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
3071 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
3072 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
3073 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
3074 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
3075 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
3076 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
3079 static struct opcode group7_rm7[] = {
3081 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
3085 static struct opcode group1[] = {
3096 static struct opcode group1A[] = {
3097 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
3100 static struct opcode group3[] = {
3101 I(DstMem | SrcImm | ModRM, em_test),
3102 I(DstMem | SrcImm | ModRM, em_test),
3103 I(DstMem | SrcNone | ModRM | Lock, em_not),
3104 I(DstMem | SrcNone | ModRM | Lock, em_neg),
3105 I(SrcMem | ModRM, em_mul_ex),
3106 I(SrcMem | ModRM, em_imul_ex),
3107 I(SrcMem | ModRM, em_div_ex),
3108 I(SrcMem | ModRM, em_idiv_ex),
3111 static struct opcode group4[] = {
3112 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
3116 static struct opcode group5[] = {
3117 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3118 D(SrcMem | ModRM | Stack),
3119 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3120 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
3121 D(SrcMem | ModRM | Stack), N,
3124 static struct opcode group6[] = {
3125 DI(ModRM | Prot, sldt),
3126 DI(ModRM | Prot, str),
3127 DI(ModRM | Prot | Priv, lldt),
3128 DI(ModRM | Prot | Priv, ltr),
3132 static struct group_dual group7 = { {
3133 DI(ModRM | Mov | DstMem | Priv, sgdt),
3134 DI(ModRM | Mov | DstMem | Priv, sidt),
3135 II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
3136 II(ModRM | SrcMem | Priv, em_lidt, lidt),
3137 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3138 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
3139 II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3141 I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
3143 N, EXT(0, group7_rm3),
3144 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3145 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3148 static struct opcode group8[] = {
3150 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
3151 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
3154 static struct group_dual group9 = { {
3155 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
3157 N, N, N, N, N, N, N, N,
3160 static struct opcode group11[] = {
3161 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
3164 static struct gprefix pfx_0f_6f_0f_7f = {
3165 N, N, N, I(Sse, em_movdqu),
3168 static struct opcode opcode_table[256] = {
3170 I6ALU(Lock, em_add),
3171 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3172 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3175 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3178 I6ALU(Lock, em_adc),
3179 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3180 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3182 I6ALU(Lock, em_sbb),
3183 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3184 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3186 I6ALU(Lock, em_and), N, N,
3188 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3190 I6ALU(Lock, em_xor), N, N,
3192 I6ALU(0, em_cmp), N, N,
3196 X8(I(SrcReg | Stack, em_push)),
3198 X8(I(DstReg | Stack, em_pop)),
3200 I(ImplicitOps | Stack | No64, em_pusha),
3201 I(ImplicitOps | Stack | No64, em_popa),
3202 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3205 I(SrcImm | Mov | Stack, em_push),
3206 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3207 I(SrcImmByte | Mov | Stack, em_push),
3208 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3209 D2bvIP(DstDI | SrcDX | Mov | String, ins, check_perm_in), /* insb, insw/insd */
3210 D2bvIP(SrcSI | DstDX | String, outs, check_perm_out), /* outsb, outsw/outsd */
3214 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
3215 G(DstMem | SrcImm | ModRM | Group, group1),
3216 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
3217 G(DstMem | SrcImmByte | ModRM | Group, group1),
3218 I2bv(DstMem | SrcReg | ModRM, em_test),
3219 I2bv(DstMem | SrcReg | ModRM | Lock, em_xchg),
3221 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
3222 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3223 I(DstMem | SrcNone | ModRM | Mov, em_mov_rm_sreg),
3224 D(ModRM | SrcMem | NoAccess | DstReg),
3225 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3228 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3230 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3231 I(SrcImmFAddr | No64, em_call_far), N,
3232 II(ImplicitOps | Stack, em_pushf, pushf),
3233 II(ImplicitOps | Stack, em_popf, popf), N, N,
3235 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3236 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
3237 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3238 I2bv(SrcSI | DstDI | String, em_cmp),
3240 I2bv(DstAcc | SrcImm, em_test),
3241 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3242 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3243 I2bv(SrcAcc | DstDI | String, em_cmp),
3245 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3247 X8(I(DstReg | SrcImm | Mov, em_mov)),
3249 D2bv(DstMem | SrcImmByte | ModRM),
3250 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3251 I(ImplicitOps | Stack, em_ret),
3252 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3253 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3254 G(ByteOp, group11), G(0, group11),
3256 N, N, N, I(ImplicitOps | Stack, em_ret_far),
3257 D(ImplicitOps), DI(SrcImmByte, intn),
3258 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3260 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3263 N, N, N, N, N, N, N, N,
3265 X3(I(SrcImmByte, em_loop)),
3266 I(SrcImmByte, em_jcxz),
3267 D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
3268 D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
3270 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
3271 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3272 D2bvIP(SrcDX | DstAcc, in, check_perm_in),
3273 D2bvIP(SrcAcc | DstDX, out, check_perm_out),
3275 N, DI(ImplicitOps, icebp), N, N,
3276 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3277 G(ByteOp, group3), G(0, group3),
3279 D(ImplicitOps), D(ImplicitOps),
3280 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3281 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3284 static struct opcode twobyte_table[256] = {
3286 G(0, group6), GD(0, &group7), N, N,
3287 N, I(ImplicitOps | VendorSpecific, em_syscall),
3288 II(ImplicitOps | Priv, em_clts, clts), N,
3289 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3290 N, D(ImplicitOps | ModRM), N, N,
3292 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3294 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3295 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3296 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3297 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3299 N, N, N, N, N, N, N, N,
3301 DI(ImplicitOps | Priv, wrmsr),
3302 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3303 DI(ImplicitOps | Priv, rdmsr),
3304 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3305 I(ImplicitOps | VendorSpecific, em_sysenter),
3306 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3308 N, N, N, N, N, N, N, N,
3310 X16(D(DstReg | SrcMem | ModRM | Mov)),
3312 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3317 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3322 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3326 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3328 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3329 DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3330 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3331 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3333 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3334 DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3335 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3336 D(DstMem | SrcReg | Src2CL | ModRM),
3337 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3339 D2bv(DstMem | SrcReg | ModRM | Lock),
3340 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3341 D(DstMem | SrcReg | ModRM | BitOp | Lock),
3342 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3343 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3344 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3347 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3348 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3349 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3351 D2bv(DstMem | SrcReg | ModRM | Lock),
3352 N, D(DstMem | SrcReg | ModRM | Mov),
3353 N, N, N, GD(0, &group9),
3354 N, N, N, N, N, N, N, N,
3356 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3358 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3360 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3376 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3380 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3386 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3387 unsigned size, bool sign_extension)
3389 int rc = X86EMUL_CONTINUE;
3393 op->addr.mem.ea = ctxt->_eip;
3394 /* NB. Immediates are sign-extended as necessary. */
3395 switch (op->bytes) {
3397 op->val = insn_fetch(s8, ctxt);
3400 op->val = insn_fetch(s16, ctxt);
3403 op->val = insn_fetch(s32, ctxt);
3406 if (!sign_extension) {
3407 switch (op->bytes) {
3415 op->val &= 0xffffffff;
3423 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
3426 int rc = X86EMUL_CONTINUE;
3430 decode_register_operand(ctxt, op,
3432 ctxt->twobyte && (ctxt->b == 0xb6 || ctxt->b == 0xb7));
3435 rc = decode_imm(ctxt, op, 1, false);
3438 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3442 if ((ctxt->d & BitOp) && op == &ctxt->dst)
3443 fetch_bit_operand(ctxt);
3444 op->orig_val = op->val;
3447 ctxt->memop.bytes = 8;
3451 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3452 op->addr.reg = &ctxt->regs[VCPU_REGS_RAX];
3453 fetch_register_operand(op);
3454 op->orig_val = op->val;
3458 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3460 register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]);
3461 op->addr.mem.seg = VCPU_SREG_ES;
3467 op->addr.reg = &ctxt->regs[VCPU_REGS_RDX];
3468 fetch_register_operand(op);
3472 op->val = ctxt->regs[VCPU_REGS_RCX] & 0xff;
3475 rc = decode_imm(ctxt, op, 1, true);
3482 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
3485 ctxt->memop.bytes = 2;
3488 ctxt->memop.bytes = 4;
3491 rc = decode_imm(ctxt, op, 2, false);
3494 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
3498 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3500 register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]);
3501 op->addr.mem.seg = seg_override(ctxt);
3506 op->addr.mem.ea = ctxt->_eip;
3507 op->bytes = ctxt->op_bytes + 2;
3508 insn_fetch_arr(op->valptr, op->bytes, ctxt);
3511 ctxt->memop.bytes = ctxt->op_bytes + 2;
3514 op->val = VCPU_SREG_ES;
3517 op->val = VCPU_SREG_CS;
3520 op->val = VCPU_SREG_SS;
3523 op->val = VCPU_SREG_DS;
3526 op->val = VCPU_SREG_FS;
3529 op->val = VCPU_SREG_GS;
3532 /* Special instructions do their own operand decoding. */
3534 op->type = OP_NONE; /* Disable writeback. */
3542 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3544 int rc = X86EMUL_CONTINUE;
3545 int mode = ctxt->mode;
3546 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3547 bool op_prefix = false;
3548 struct opcode opcode;
3550 ctxt->memop.type = OP_NONE;
3551 ctxt->memopp = NULL;
3552 ctxt->_eip = ctxt->eip;
3553 ctxt->fetch.start = ctxt->_eip;
3554 ctxt->fetch.end = ctxt->fetch.start + insn_len;
3556 memcpy(ctxt->fetch.data, insn, insn_len);
3559 case X86EMUL_MODE_REAL:
3560 case X86EMUL_MODE_VM86:
3561 case X86EMUL_MODE_PROT16:
3562 def_op_bytes = def_ad_bytes = 2;
3564 case X86EMUL_MODE_PROT32:
3565 def_op_bytes = def_ad_bytes = 4;
3567 #ifdef CONFIG_X86_64
3568 case X86EMUL_MODE_PROT64:
3574 return EMULATION_FAILED;
3577 ctxt->op_bytes = def_op_bytes;
3578 ctxt->ad_bytes = def_ad_bytes;
3580 /* Legacy prefixes. */
3582 switch (ctxt->b = insn_fetch(u8, ctxt)) {
3583 case 0x66: /* operand-size override */
3585 /* switch between 2/4 bytes */
3586 ctxt->op_bytes = def_op_bytes ^ 6;
3588 case 0x67: /* address-size override */
3589 if (mode == X86EMUL_MODE_PROT64)
3590 /* switch between 4/8 bytes */
3591 ctxt->ad_bytes = def_ad_bytes ^ 12;
3593 /* switch between 2/4 bytes */
3594 ctxt->ad_bytes = def_ad_bytes ^ 6;
3596 case 0x26: /* ES override */
3597 case 0x2e: /* CS override */
3598 case 0x36: /* SS override */
3599 case 0x3e: /* DS override */
3600 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
3602 case 0x64: /* FS override */
3603 case 0x65: /* GS override */
3604 set_seg_override(ctxt, ctxt->b & 7);
3606 case 0x40 ... 0x4f: /* REX */
3607 if (mode != X86EMUL_MODE_PROT64)
3609 ctxt->rex_prefix = ctxt->b;
3611 case 0xf0: /* LOCK */
3612 ctxt->lock_prefix = 1;
3614 case 0xf2: /* REPNE/REPNZ */
3615 case 0xf3: /* REP/REPE/REPZ */
3616 ctxt->rep_prefix = ctxt->b;
3622 /* Any legacy prefix after a REX prefix nullifies its effect. */
3624 ctxt->rex_prefix = 0;
3630 if (ctxt->rex_prefix & 8)
3631 ctxt->op_bytes = 8; /* REX.W */
3633 /* Opcode byte(s). */
3634 opcode = opcode_table[ctxt->b];
3635 /* Two-byte opcode? */
3636 if (ctxt->b == 0x0f) {
3638 ctxt->b = insn_fetch(u8, ctxt);
3639 opcode = twobyte_table[ctxt->b];
3641 ctxt->d = opcode.flags;
3643 while (ctxt->d & GroupMask) {
3644 switch (ctxt->d & GroupMask) {
3646 ctxt->modrm = insn_fetch(u8, ctxt);
3648 goffset = (ctxt->modrm >> 3) & 7;
3649 opcode = opcode.u.group[goffset];
3652 ctxt->modrm = insn_fetch(u8, ctxt);
3654 goffset = (ctxt->modrm >> 3) & 7;
3655 if ((ctxt->modrm >> 6) == 3)
3656 opcode = opcode.u.gdual->mod3[goffset];
3658 opcode = opcode.u.gdual->mod012[goffset];
3661 goffset = ctxt->modrm & 7;
3662 opcode = opcode.u.group[goffset];
3665 if (ctxt->rep_prefix && op_prefix)
3666 return EMULATION_FAILED;
3667 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
3668 switch (simd_prefix) {
3669 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3670 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3671 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3672 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3676 return EMULATION_FAILED;
3679 ctxt->d &= ~(u64)GroupMask;
3680 ctxt->d |= opcode.flags;
3683 ctxt->execute = opcode.u.execute;
3684 ctxt->check_perm = opcode.check_perm;
3685 ctxt->intercept = opcode.intercept;
3688 if (ctxt->d == 0 || (ctxt->d & Undefined))
3689 return EMULATION_FAILED;
3691 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3692 return EMULATION_FAILED;
3694 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
3697 if (ctxt->d & Op3264) {
3698 if (mode == X86EMUL_MODE_PROT64)
3705 ctxt->op_bytes = 16;
3707 /* ModRM and SIB bytes. */
3708 if (ctxt->d & ModRM) {
3709 rc = decode_modrm(ctxt, &ctxt->memop);
3710 if (!ctxt->has_seg_override)
3711 set_seg_override(ctxt, ctxt->modrm_seg);
3712 } else if (ctxt->d & MemAbs)
3713 rc = decode_abs(ctxt, &ctxt->memop);
3714 if (rc != X86EMUL_CONTINUE)
3717 if (!ctxt->has_seg_override)
3718 set_seg_override(ctxt, VCPU_SREG_DS);
3720 ctxt->memop.addr.mem.seg = seg_override(ctxt);
3722 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
3723 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
3726 * Decode and fetch the source operand: register, memory
3729 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
3730 if (rc != X86EMUL_CONTINUE)
3734 * Decode and fetch the second source operand: register, memory
3737 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
3738 if (rc != X86EMUL_CONTINUE)
3741 /* Decode and fetch the destination operand: register or memory. */
3742 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
3745 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
3746 ctxt->memopp->addr.mem.ea += ctxt->_eip;
3748 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
3751 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3753 /* The second termination condition only applies for REPE
3754 * and REPNE. Test if the repeat string operation prefix is
3755 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3756 * corresponding termination condition according to:
3757 * - if REPE/REPZ and ZF = 0 then done
3758 * - if REPNE/REPNZ and ZF = 1 then done
3760 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
3761 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
3762 && (((ctxt->rep_prefix == REPE_PREFIX) &&
3763 ((ctxt->eflags & EFLG_ZF) == 0))
3764 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
3765 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3771 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3773 struct x86_emulate_ops *ops = ctxt->ops;
3775 int rc = X86EMUL_CONTINUE;
3776 int saved_dst_type = ctxt->dst.type;
3778 ctxt->mem_read.pos = 0;
3780 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
3781 rc = emulate_ud(ctxt);
3785 /* LOCK prefix is allowed only with some instructions */
3786 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
3787 rc = emulate_ud(ctxt);
3791 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
3792 rc = emulate_ud(ctxt);
3797 && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
3798 || !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
3799 rc = emulate_ud(ctxt);
3803 if ((ctxt->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
3804 rc = emulate_nm(ctxt);
3808 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3809 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3810 X86_ICPT_PRE_EXCEPT);
3811 if (rc != X86EMUL_CONTINUE)
3815 /* Privileged instruction can be executed only in CPL=0 */
3816 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
3817 rc = emulate_gp(ctxt, 0);
3821 /* Instruction can only be executed in protected mode */
3822 if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3823 rc = emulate_ud(ctxt);
3827 /* Do instruction specific permission checks */
3828 if (ctxt->check_perm) {
3829 rc = ctxt->check_perm(ctxt);
3830 if (rc != X86EMUL_CONTINUE)
3834 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3835 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3836 X86_ICPT_POST_EXCEPT);
3837 if (rc != X86EMUL_CONTINUE)
3841 if (ctxt->rep_prefix && (ctxt->d & String)) {
3842 /* All REP prefixes have the same first termination condition */
3843 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) {
3844 ctxt->eip = ctxt->_eip;
3849 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
3850 rc = segmented_read(ctxt, ctxt->src.addr.mem,
3851 ctxt->src.valptr, ctxt->src.bytes);
3852 if (rc != X86EMUL_CONTINUE)
3854 ctxt->src.orig_val64 = ctxt->src.val64;
3857 if (ctxt->src2.type == OP_MEM) {
3858 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
3859 &ctxt->src2.val, ctxt->src2.bytes);
3860 if (rc != X86EMUL_CONTINUE)
3864 if ((ctxt->d & DstMask) == ImplicitOps)
3868 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
3869 /* optimisation - avoid slow emulated read if Mov */
3870 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
3871 &ctxt->dst.val, ctxt->dst.bytes);
3872 if (rc != X86EMUL_CONTINUE)
3875 ctxt->dst.orig_val = ctxt->dst.val;
3879 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3880 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3881 X86_ICPT_POST_MEMACCESS);
3882 if (rc != X86EMUL_CONTINUE)
3886 if (ctxt->execute) {
3887 rc = ctxt->execute(ctxt);
3888 if (rc != X86EMUL_CONTINUE)
3897 case 0x40 ... 0x47: /* inc r16/r32 */
3898 emulate_1op(ctxt, "inc");
3900 case 0x48 ... 0x4f: /* dec r16/r32 */
3901 emulate_1op(ctxt, "dec");
3903 case 0x63: /* movsxd */
3904 if (ctxt->mode != X86EMUL_MODE_PROT64)
3905 goto cannot_emulate;
3906 ctxt->dst.val = (s32) ctxt->src.val;
3908 case 0x6c: /* insb */
3909 case 0x6d: /* insw/insd */
3910 ctxt->src.val = ctxt->regs[VCPU_REGS_RDX];
3912 case 0x6e: /* outsb */
3913 case 0x6f: /* outsw/outsd */
3914 ctxt->dst.val = ctxt->regs[VCPU_REGS_RDX];
3917 case 0x70 ... 0x7f: /* jcc (short) */
3918 if (test_cc(ctxt->b, ctxt->eflags))
3919 jmp_rel(ctxt, ctxt->src.val);
3921 case 0x8d: /* lea r16/r32, m */
3922 ctxt->dst.val = ctxt->src.addr.mem.ea;
3924 case 0x8f: /* pop (sole member of Grp1a) */
3925 rc = em_grp1a(ctxt);
3927 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3928 if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX])
3932 case 0x98: /* cbw/cwde/cdqe */
3933 switch (ctxt->op_bytes) {
3934 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
3935 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
3936 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
3942 case 0xcc: /* int3 */
3943 rc = emulate_int(ctxt, 3);
3945 case 0xcd: /* int n */
3946 rc = emulate_int(ctxt, ctxt->src.val);
3948 case 0xce: /* into */
3949 if (ctxt->eflags & EFLG_OF)
3950 rc = emulate_int(ctxt, 4);
3952 case 0xd0 ... 0xd1: /* Grp2 */
3955 case 0xd2 ... 0xd3: /* Grp2 */
3956 ctxt->src.val = ctxt->regs[VCPU_REGS_RCX];
3959 case 0xe4: /* inb */
3962 case 0xe6: /* outb */
3963 case 0xe7: /* out */
3965 case 0xe8: /* call (near) */ {
3966 long int rel = ctxt->src.val;
3967 ctxt->src.val = (unsigned long) ctxt->_eip;
3972 case 0xe9: /* jmp rel */
3973 case 0xeb: /* jmp rel short */
3974 jmp_rel(ctxt, ctxt->src.val);
3975 ctxt->dst.type = OP_NONE; /* Disable writeback. */
3977 case 0xec: /* in al,dx */
3978 case 0xed: /* in (e/r)ax,dx */
3980 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3982 goto done; /* IO is needed */
3984 case 0xee: /* out dx,al */
3985 case 0xef: /* out dx,(e/r)ax */
3987 ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3989 ctxt->dst.type = OP_NONE; /* Disable writeback. */
3991 case 0xf4: /* hlt */
3992 ctxt->ops->halt(ctxt);
3994 case 0xf5: /* cmc */
3995 /* complement carry flag from eflags reg */
3996 ctxt->eflags ^= EFLG_CF;
3998 case 0xf8: /* clc */
3999 ctxt->eflags &= ~EFLG_CF;
4001 case 0xf9: /* stc */
4002 ctxt->eflags |= EFLG_CF;
4004 case 0xfc: /* cld */
4005 ctxt->eflags &= ~EFLG_DF;
4007 case 0xfd: /* std */
4008 ctxt->eflags |= EFLG_DF;
4010 case 0xfe: /* Grp4 */
4011 rc = em_grp45(ctxt);
4013 case 0xff: /* Grp5 */
4014 rc = em_grp45(ctxt);
4017 goto cannot_emulate;
4020 if (rc != X86EMUL_CONTINUE)
4024 rc = writeback(ctxt);
4025 if (rc != X86EMUL_CONTINUE)
4029 * restore dst type in case the decoding will be reused
4030 * (happens for string instruction )
4032 ctxt->dst.type = saved_dst_type;
4034 if ((ctxt->d & SrcMask) == SrcSI)
4035 string_addr_inc(ctxt, seg_override(ctxt),
4036 VCPU_REGS_RSI, &ctxt->src);
4038 if ((ctxt->d & DstMask) == DstDI)
4039 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4042 if (ctxt->rep_prefix && (ctxt->d & String)) {
4043 struct read_cache *r = &ctxt->io_read;
4044 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
4046 if (!string_insn_completed(ctxt)) {
4048 * Re-enter guest when pio read ahead buffer is empty
4049 * or, if it is not used, after each 1024 iteration.
4051 if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) &&
4052 (r->end == 0 || r->end != r->pos)) {
4054 * Reset read cache. Usually happens before
4055 * decode, but since instruction is restarted
4056 * we have to do it here.
4058 ctxt->mem_read.end = 0;
4059 return EMULATION_RESTART;
4061 goto done; /* skip rip writeback */
4065 ctxt->eip = ctxt->_eip;
4068 if (rc == X86EMUL_PROPAGATE_FAULT)
4069 ctxt->have_exception = true;
4070 if (rc == X86EMUL_INTERCEPTED)
4071 return EMULATION_INTERCEPTED;
4073 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4077 case 0x09: /* wbinvd */
4078 (ctxt->ops->wbinvd)(ctxt);
4080 case 0x08: /* invd */
4081 case 0x0d: /* GrpP (prefetch) */
4082 case 0x18: /* Grp16 (prefetch/nop) */
4084 case 0x20: /* mov cr, reg */
4085 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4087 case 0x21: /* mov from dr to reg */
4088 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4090 case 0x22: /* mov reg, cr */
4091 if (ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val)) {
4092 emulate_gp(ctxt, 0);
4093 rc = X86EMUL_PROPAGATE_FAULT;
4096 ctxt->dst.type = OP_NONE;
4098 case 0x23: /* mov from reg to dr */
4099 if (ops->set_dr(ctxt, ctxt->modrm_reg, ctxt->src.val &
4100 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4101 ~0ULL : ~0U)) < 0) {
4102 /* #UD condition is already handled by the code above */
4103 emulate_gp(ctxt, 0);
4104 rc = X86EMUL_PROPAGATE_FAULT;
4108 ctxt->dst.type = OP_NONE; /* no writeback */
4112 msr_data = (u32)ctxt->regs[VCPU_REGS_RAX]
4113 | ((u64)ctxt->regs[VCPU_REGS_RDX] << 32);
4114 if (ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data)) {
4115 emulate_gp(ctxt, 0);
4116 rc = X86EMUL_PROPAGATE_FAULT;
4119 rc = X86EMUL_CONTINUE;
4123 if (ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data)) {
4124 emulate_gp(ctxt, 0);
4125 rc = X86EMUL_PROPAGATE_FAULT;
4128 ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data;
4129 ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32;
4131 rc = X86EMUL_CONTINUE;
4133 case 0x40 ... 0x4f: /* cmov */
4134 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4135 if (!test_cc(ctxt->b, ctxt->eflags))
4136 ctxt->dst.type = OP_NONE; /* no writeback */
4138 case 0x80 ... 0x8f: /* jnz rel, etc*/
4139 if (test_cc(ctxt->b, ctxt->eflags))
4140 jmp_rel(ctxt, ctxt->src.val);
4142 case 0x90 ... 0x9f: /* setcc r/m8 */
4143 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4147 ctxt->dst.type = OP_NONE;
4148 /* only subword offset */
4149 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
4150 emulate_2op_SrcV_nobyte(ctxt, "bt");
4152 case 0xa4: /* shld imm8, r, r/m */
4153 case 0xa5: /* shld cl, r, r/m */
4154 emulate_2op_cl(ctxt, "shld");
4158 emulate_2op_SrcV_nobyte(ctxt, "bts");
4160 case 0xac: /* shrd imm8, r, r/m */
4161 case 0xad: /* shrd cl, r, r/m */
4162 emulate_2op_cl(ctxt, "shrd");
4164 case 0xae: /* clflush */
4166 case 0xb0 ... 0xb1: /* cmpxchg */
4168 * Save real source value, then compare EAX against
4171 ctxt->src.orig_val = ctxt->src.val;
4172 ctxt->src.val = ctxt->regs[VCPU_REGS_RAX];
4173 emulate_2op_SrcV(ctxt, "cmp");
4174 if (ctxt->eflags & EFLG_ZF) {
4175 /* Success: write back to memory. */
4176 ctxt->dst.val = ctxt->src.orig_val;
4178 /* Failure: write the value we saw to EAX. */
4179 ctxt->dst.type = OP_REG;
4180 ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX];
4185 emulate_2op_SrcV_nobyte(ctxt, "btr");
4187 case 0xb6 ... 0xb7: /* movzx */
4188 ctxt->dst.bytes = ctxt->op_bytes;
4189 ctxt->dst.val = (ctxt->d & ByteOp) ? (u8) ctxt->src.val
4190 : (u16) ctxt->src.val;
4192 case 0xba: /* Grp8 */
4193 switch (ctxt->modrm_reg & 3) {
4206 emulate_2op_SrcV_nobyte(ctxt, "btc");
4208 case 0xbc: { /* bsf */
4210 __asm__ ("bsf %2, %0; setz %1"
4211 : "=r"(ctxt->dst.val), "=q"(zf)
4212 : "r"(ctxt->src.val));
4213 ctxt->eflags &= ~X86_EFLAGS_ZF;
4215 ctxt->eflags |= X86_EFLAGS_ZF;
4216 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4220 case 0xbd: { /* bsr */
4222 __asm__ ("bsr %2, %0; setz %1"
4223 : "=r"(ctxt->dst.val), "=q"(zf)
4224 : "r"(ctxt->src.val));
4225 ctxt->eflags &= ~X86_EFLAGS_ZF;
4227 ctxt->eflags |= X86_EFLAGS_ZF;
4228 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4232 case 0xbe ... 0xbf: /* movsx */
4233 ctxt->dst.bytes = ctxt->op_bytes;
4234 ctxt->dst.val = (ctxt->d & ByteOp) ? (s8) ctxt->src.val :
4235 (s16) ctxt->src.val;
4237 case 0xc0 ... 0xc1: /* xadd */
4238 emulate_2op_SrcV(ctxt, "add");
4239 /* Write back the register source. */
4240 ctxt->src.val = ctxt->dst.orig_val;
4241 write_register_operand(&ctxt->src);
4243 case 0xc3: /* movnti */
4244 ctxt->dst.bytes = ctxt->op_bytes;
4245 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4246 (u64) ctxt->src.val;
4248 case 0xc7: /* Grp9 (cmpxchg8b) */
4252 goto cannot_emulate;
4255 if (rc != X86EMUL_CONTINUE)
4261 return EMULATION_FAILED;