1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
35 #define OpImplicit 1ull /* No generic decode */
36 #define OpReg 2ull /* Register */
37 #define OpMem 3ull /* Memory */
38 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
39 #define OpDI 5ull /* ES:DI/EDI/RDI */
40 #define OpMem64 6ull /* Memory, 64-bit */
41 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
42 #define OpDX 8ull /* DX register */
43 #define OpCL 9ull /* CL register (for shifts) */
44 #define OpImmByte 10ull /* 8-bit sign extended immediate */
45 #define OpOne 11ull /* Implied 1 */
46 #define OpImm 12ull /* Sign extended immediate */
47 #define OpMem16 13ull /* Memory operand (16-bit). */
48 #define OpMem32 14ull /* Memory operand (32-bit). */
49 #define OpImmU 15ull /* Immediate operand, zero extended */
50 #define OpSI 16ull /* SI/ESI/RSI */
51 #define OpImmFAddr 17ull /* Immediate far address */
52 #define OpMemFAddr 18ull /* Far address in memory */
53 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
54 #define OpES 20ull /* ES */
55 #define OpCS 21ull /* CS */
56 #define OpSS 22ull /* SS */
57 #define OpDS 23ull /* DS */
58 #define OpFS 24ull /* FS */
59 #define OpGS 25ull /* GS */
61 #define OpBits 5 /* Width of operand field */
62 #define OpMask ((1ull << OpBits) - 1)
65 * Opcode effective-address decode tables.
66 * Note that we only emulate instructions that have at least one memory
67 * operand (excluding implicit stack references). We assume that stack
68 * references and instruction fetches will never occur in special memory
69 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
73 /* Operand sizes: 8-bit operands or specified/overridden size. */
74 #define ByteOp (1<<0) /* 8-bit operands. */
75 /* Destination operand type. */
77 #define ImplicitOps (OpImplicit << DstShift)
78 #define DstReg (OpReg << DstShift)
79 #define DstMem (OpMem << DstShift)
80 #define DstAcc (OpAcc << DstShift)
81 #define DstDI (OpDI << DstShift)
82 #define DstMem64 (OpMem64 << DstShift)
83 #define DstImmUByte (OpImmUByte << DstShift)
84 #define DstDX (OpDX << DstShift)
85 #define DstMask (OpMask << DstShift)
86 /* Source operand type. */
88 #define SrcNone (OpNone << SrcShift)
89 #define SrcReg (OpReg << SrcShift)
90 #define SrcMem (OpMem << SrcShift)
91 #define SrcMem16 (OpMem16 << SrcShift)
92 #define SrcMem32 (OpMem32 << SrcShift)
93 #define SrcImm (OpImm << SrcShift)
94 #define SrcImmByte (OpImmByte << SrcShift)
95 #define SrcOne (OpOne << SrcShift)
96 #define SrcImmUByte (OpImmUByte << SrcShift)
97 #define SrcImmU (OpImmU << SrcShift)
98 #define SrcSI (OpSI << SrcShift)
99 #define SrcImmFAddr (OpImmFAddr << SrcShift)
100 #define SrcMemFAddr (OpMemFAddr << SrcShift)
101 #define SrcAcc (OpAcc << SrcShift)
102 #define SrcImmU16 (OpImmU16 << SrcShift)
103 #define SrcDX (OpDX << SrcShift)
104 #define SrcMask (OpMask << SrcShift)
105 #define BitOp (1<<11)
106 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
107 #define String (1<<13) /* String instruction (rep capable) */
108 #define Stack (1<<14) /* Stack instruction (push/pop) */
109 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
110 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
111 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
112 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
113 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
114 #define Sse (1<<18) /* SSE Vector instruction */
115 /* Generic ModRM decode. */
116 #define ModRM (1<<19)
117 /* Destination is only written; never read. */
120 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
121 #define VendorSpecific (1<<22) /* Vendor specific instruction */
122 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
123 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
124 #define Undefined (1<<25) /* No Such Instruction */
125 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
126 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
128 /* Source 2 operand type */
129 #define Src2Shift (29)
130 #define Src2None (OpNone << Src2Shift)
131 #define Src2CL (OpCL << Src2Shift)
132 #define Src2ImmByte (OpImmByte << Src2Shift)
133 #define Src2One (OpOne << Src2Shift)
134 #define Src2Imm (OpImm << Src2Shift)
135 #define Src2ES (OpES << Src2Shift)
136 #define Src2CS (OpCS << Src2Shift)
137 #define Src2SS (OpSS << Src2Shift)
138 #define Src2DS (OpDS << Src2Shift)
139 #define Src2FS (OpFS << Src2Shift)
140 #define Src2GS (OpGS << Src2Shift)
141 #define Src2Mask (OpMask << Src2Shift)
143 #define X2(x...) x, x
144 #define X3(x...) X2(x), x
145 #define X4(x...) X2(x), X2(x)
146 #define X5(x...) X4(x), x
147 #define X6(x...) X4(x), X2(x)
148 #define X7(x...) X4(x), X3(x)
149 #define X8(x...) X4(x), X4(x)
150 #define X16(x...) X8(x), X8(x)
156 int (*execute)(struct x86_emulate_ctxt *ctxt);
157 struct opcode *group;
158 struct group_dual *gdual;
159 struct gprefix *gprefix;
161 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
165 struct opcode mod012[8];
166 struct opcode mod3[8];
170 struct opcode pfx_no;
171 struct opcode pfx_66;
172 struct opcode pfx_f2;
173 struct opcode pfx_f3;
176 /* EFLAGS bit definitions. */
177 #define EFLG_ID (1<<21)
178 #define EFLG_VIP (1<<20)
179 #define EFLG_VIF (1<<19)
180 #define EFLG_AC (1<<18)
181 #define EFLG_VM (1<<17)
182 #define EFLG_RF (1<<16)
183 #define EFLG_IOPL (3<<12)
184 #define EFLG_NT (1<<14)
185 #define EFLG_OF (1<<11)
186 #define EFLG_DF (1<<10)
187 #define EFLG_IF (1<<9)
188 #define EFLG_TF (1<<8)
189 #define EFLG_SF (1<<7)
190 #define EFLG_ZF (1<<6)
191 #define EFLG_AF (1<<4)
192 #define EFLG_PF (1<<2)
193 #define EFLG_CF (1<<0)
195 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
196 #define EFLG_RESERVED_ONE_MASK 2
199 * Instruction emulation:
200 * Most instructions are emulated directly via a fragment of inline assembly
201 * code. This allows us to save/restore EFLAGS and thus very easily pick up
202 * any modified flags.
205 #if defined(CONFIG_X86_64)
206 #define _LO32 "k" /* force 32-bit operand */
207 #define _STK "%%rsp" /* stack pointer */
208 #elif defined(__i386__)
209 #define _LO32 "" /* force 32-bit operand */
210 #define _STK "%%esp" /* stack pointer */
214 * These EFLAGS bits are restored from saved value during emulation, and
215 * any changes are written back to the saved value after emulation.
217 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
219 /* Before executing instruction: restore necessary bits in EFLAGS. */
220 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
221 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
222 "movl %"_sav",%"_LO32 _tmp"; " \
225 "movl %"_msk",%"_LO32 _tmp"; " \
226 "andl %"_LO32 _tmp",("_STK"); " \
228 "notl %"_LO32 _tmp"; " \
229 "andl %"_LO32 _tmp",("_STK"); " \
230 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
232 "orl %"_LO32 _tmp",("_STK"); " \
236 /* After executing instruction: write-back necessary bits in EFLAGS. */
237 #define _POST_EFLAGS(_sav, _msk, _tmp) \
238 /* _sav |= EFLAGS & _msk; */ \
241 "andl %"_msk",%"_LO32 _tmp"; " \
242 "orl %"_LO32 _tmp",%"_sav"; "
250 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
252 __asm__ __volatile__ ( \
253 _PRE_EFLAGS("0", "4", "2") \
254 _op _suffix " %"_x"3,%1; " \
255 _POST_EFLAGS("0", "4", "2") \
256 : "=m" ((ctxt)->eflags), \
257 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
259 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
263 /* Raw emulation: instruction has two explicit operands. */
264 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
266 unsigned long _tmp; \
268 switch ((ctxt)->dst.bytes) { \
270 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
273 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
276 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
281 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
283 unsigned long _tmp; \
284 switch ((ctxt)->dst.bytes) { \
286 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
289 __emulate_2op_nobyte(ctxt, _op, \
290 _wx, _wy, _lx, _ly, _qx, _qy); \
295 /* Source operand is byte-sized and may be restricted to just %cl. */
296 #define emulate_2op_SrcB(ctxt, _op) \
297 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
299 /* Source operand is byte, word, long or quad sized. */
300 #define emulate_2op_SrcV(ctxt, _op) \
301 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
303 /* Source operand is word, long or quad sized. */
304 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
305 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
307 /* Instruction has three operands and one operand is stored in ECX register */
308 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
310 unsigned long _tmp; \
311 _type _clv = (ctxt)->src2.val; \
312 _type _srcv = (ctxt)->src.val; \
313 _type _dstv = (ctxt)->dst.val; \
315 __asm__ __volatile__ ( \
316 _PRE_EFLAGS("0", "5", "2") \
317 _op _suffix " %4,%1 \n" \
318 _POST_EFLAGS("0", "5", "2") \
319 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
320 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
323 (ctxt)->src2.val = (unsigned long) _clv; \
324 (ctxt)->src2.val = (unsigned long) _srcv; \
325 (ctxt)->dst.val = (unsigned long) _dstv; \
328 #define emulate_2op_cl(ctxt, _op) \
330 switch ((ctxt)->dst.bytes) { \
332 __emulate_2op_cl(ctxt, _op, "w", u16); \
335 __emulate_2op_cl(ctxt, _op, "l", u32); \
338 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
343 #define __emulate_1op(ctxt, _op, _suffix) \
345 unsigned long _tmp; \
347 __asm__ __volatile__ ( \
348 _PRE_EFLAGS("0", "3", "2") \
349 _op _suffix " %1; " \
350 _POST_EFLAGS("0", "3", "2") \
351 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
353 : "i" (EFLAGS_MASK)); \
356 /* Instruction has only one explicit operand (no source operand). */
357 #define emulate_1op(ctxt, _op) \
359 switch ((ctxt)->dst.bytes) { \
360 case 1: __emulate_1op(ctxt, _op, "b"); break; \
361 case 2: __emulate_1op(ctxt, _op, "w"); break; \
362 case 4: __emulate_1op(ctxt, _op, "l"); break; \
363 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
367 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
369 unsigned long _tmp; \
370 ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX]; \
371 ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX]; \
373 __asm__ __volatile__ ( \
374 _PRE_EFLAGS("0", "5", "1") \
376 _op _suffix " %6; " \
378 _POST_EFLAGS("0", "5", "1") \
379 ".pushsection .fixup,\"ax\" \n\t" \
380 "3: movb $1, %4 \n\t" \
383 _ASM_EXTABLE(1b, 3b) \
384 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
385 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
386 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val), \
387 "a" (*rax), "d" (*rdx)); \
390 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
391 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
393 switch((ctxt)->src.bytes) { \
395 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
398 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
401 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
404 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
409 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
410 enum x86_intercept intercept,
411 enum x86_intercept_stage stage)
413 struct x86_instruction_info info = {
414 .intercept = intercept,
415 .rep_prefix = ctxt->rep_prefix,
416 .modrm_mod = ctxt->modrm_mod,
417 .modrm_reg = ctxt->modrm_reg,
418 .modrm_rm = ctxt->modrm_rm,
419 .src_val = ctxt->src.val64,
420 .src_bytes = ctxt->src.bytes,
421 .dst_bytes = ctxt->dst.bytes,
422 .ad_bytes = ctxt->ad_bytes,
423 .next_rip = ctxt->eip,
426 return ctxt->ops->intercept(ctxt, &info, stage);
429 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
431 return (1UL << (ctxt->ad_bytes << 3)) - 1;
434 /* Access/update address held in a register, based on addressing mode. */
435 static inline unsigned long
436 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
438 if (ctxt->ad_bytes == sizeof(unsigned long))
441 return reg & ad_mask(ctxt);
444 static inline unsigned long
445 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
447 return address_mask(ctxt, reg);
451 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
453 if (ctxt->ad_bytes == sizeof(unsigned long))
456 *reg = (*reg & ~ad_mask(ctxt)) | ((*reg + inc) & ad_mask(ctxt));
459 static u32 desc_limit_scaled(struct desc_struct *desc)
461 u32 limit = get_desc_limit(desc);
463 return desc->g ? (limit << 12) | 0xfff : limit;
466 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
468 ctxt->has_seg_override = true;
469 ctxt->seg_override = seg;
472 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
474 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
477 return ctxt->ops->get_cached_segment_base(ctxt, seg);
480 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
482 if (!ctxt->has_seg_override)
485 return ctxt->seg_override;
488 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
489 u32 error, bool valid)
491 ctxt->exception.vector = vec;
492 ctxt->exception.error_code = error;
493 ctxt->exception.error_code_valid = valid;
494 return X86EMUL_PROPAGATE_FAULT;
497 static int emulate_db(struct x86_emulate_ctxt *ctxt)
499 return emulate_exception(ctxt, DB_VECTOR, 0, false);
502 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
504 return emulate_exception(ctxt, GP_VECTOR, err, true);
507 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
509 return emulate_exception(ctxt, SS_VECTOR, err, true);
512 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
514 return emulate_exception(ctxt, UD_VECTOR, 0, false);
517 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
519 return emulate_exception(ctxt, TS_VECTOR, err, true);
522 static int emulate_de(struct x86_emulate_ctxt *ctxt)
524 return emulate_exception(ctxt, DE_VECTOR, 0, false);
527 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
529 return emulate_exception(ctxt, NM_VECTOR, 0, false);
532 static inline int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
535 switch (ctxt->op_bytes) {
537 ctxt->_eip = (u16)dst;
540 ctxt->_eip = (u32)dst;
543 if ((cs_l && is_noncanonical_address(dst)) ||
544 (!cs_l && (dst & ~(u32)-1)))
545 return emulate_gp(ctxt, 0);
549 WARN(1, "unsupported eip assignment size\n");
551 return X86EMUL_CONTINUE;
554 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
556 return assign_eip_far(ctxt, dst, ctxt->mode == X86EMUL_MODE_PROT64);
559 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
561 return assign_eip_near(ctxt, ctxt->_eip + rel);
564 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
567 struct desc_struct desc;
569 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
573 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
578 struct desc_struct desc;
580 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
581 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
584 static int __linearize(struct x86_emulate_ctxt *ctxt,
585 struct segmented_address addr,
586 unsigned size, bool write, bool fetch,
589 struct desc_struct desc;
596 la = seg_base(ctxt, addr.seg) + addr.ea;
597 switch (ctxt->mode) {
598 case X86EMUL_MODE_REAL:
600 case X86EMUL_MODE_PROT64:
601 if (((signed long)la << 16) >> 16 != la)
602 return emulate_gp(ctxt, 0);
605 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
609 /* code segment or read-only data segment */
610 if (((desc.type & 8) || !(desc.type & 2)) && write)
612 /* unreadable code segment */
613 if (!fetch && (desc.type & 8) && !(desc.type & 2))
615 lim = desc_limit_scaled(&desc);
616 if ((desc.type & 8) || !(desc.type & 4)) {
617 /* expand-up segment */
618 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
621 /* exapand-down segment */
622 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
624 lim = desc.d ? 0xffffffff : 0xffff;
625 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
628 cpl = ctxt->ops->cpl(ctxt);
631 if (!(desc.type & 8)) {
635 } else if ((desc.type & 8) && !(desc.type & 4)) {
636 /* nonconforming code segment */
639 } else if ((desc.type & 8) && (desc.type & 4)) {
640 /* conforming code segment */
646 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
649 return X86EMUL_CONTINUE;
651 if (addr.seg == VCPU_SREG_SS)
652 return emulate_ss(ctxt, addr.seg);
654 return emulate_gp(ctxt, addr.seg);
657 static int linearize(struct x86_emulate_ctxt *ctxt,
658 struct segmented_address addr,
659 unsigned size, bool write,
662 return __linearize(ctxt, addr, size, write, false, linear);
666 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
667 struct segmented_address addr,
674 rc = linearize(ctxt, addr, size, false, &linear);
675 if (rc != X86EMUL_CONTINUE)
677 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
681 * Fetch the next byte of the instruction being emulated which is pointed to
682 * by ctxt->_eip, then increment ctxt->_eip.
684 * Also prefetch the remaining bytes of the instruction without crossing page
685 * boundary if they are not in fetch_cache yet.
687 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
689 struct fetch_cache *fc = &ctxt->fetch;
693 if (ctxt->_eip == fc->end) {
694 unsigned long linear;
695 struct segmented_address addr = { .seg = VCPU_SREG_CS,
697 cur_size = fc->end - fc->start;
698 size = min(15UL - cur_size,
699 PAGE_SIZE - offset_in_page(ctxt->_eip));
700 rc = __linearize(ctxt, addr, size, false, true, &linear);
701 if (unlikely(rc != X86EMUL_CONTINUE))
703 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
704 size, &ctxt->exception);
705 if (unlikely(rc != X86EMUL_CONTINUE))
709 *dest = fc->data[ctxt->_eip - fc->start];
711 return X86EMUL_CONTINUE;
714 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
715 void *dest, unsigned size)
719 /* x86 instructions are limited to 15 bytes. */
720 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
721 return X86EMUL_UNHANDLEABLE;
723 rc = do_insn_fetch_byte(ctxt, dest++);
724 if (rc != X86EMUL_CONTINUE)
727 return X86EMUL_CONTINUE;
730 /* Fetch next part of the instruction being emulated. */
731 #define insn_fetch(_type, _ctxt) \
732 ({ unsigned long _x; \
733 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
734 if (rc != X86EMUL_CONTINUE) \
739 #define insn_fetch_arr(_arr, _size, _ctxt) \
740 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
741 if (rc != X86EMUL_CONTINUE) \
746 * Given the 'reg' portion of a ModRM byte, and a register block, return a
747 * pointer into the block that addresses the relevant register.
748 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
750 static void *decode_register(u8 modrm_reg, unsigned long *regs,
755 p = ®s[modrm_reg];
756 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
757 p = (unsigned char *)®s[modrm_reg & 3] + 1;
761 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
762 struct segmented_address addr,
763 u16 *size, unsigned long *address, int op_bytes)
770 rc = segmented_read_std(ctxt, addr, size, 2);
771 if (rc != X86EMUL_CONTINUE)
774 rc = segmented_read_std(ctxt, addr, address, op_bytes);
778 static int test_cc(unsigned int condition, unsigned int flags)
782 switch ((condition & 15) >> 1) {
784 rc |= (flags & EFLG_OF);
786 case 1: /* b/c/nae */
787 rc |= (flags & EFLG_CF);
790 rc |= (flags & EFLG_ZF);
793 rc |= (flags & (EFLG_CF|EFLG_ZF));
796 rc |= (flags & EFLG_SF);
799 rc |= (flags & EFLG_PF);
802 rc |= (flags & EFLG_ZF);
805 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
809 /* Odd condition identifiers (lsb == 1) have inverted sense. */
810 return (!!rc ^ (condition & 1));
813 static void fetch_register_operand(struct operand *op)
817 op->val = *(u8 *)op->addr.reg;
820 op->val = *(u16 *)op->addr.reg;
823 op->val = *(u32 *)op->addr.reg;
826 op->val = *(u64 *)op->addr.reg;
831 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
833 ctxt->ops->get_fpu(ctxt);
835 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
836 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
837 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
838 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
839 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
840 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
841 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
842 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
844 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
845 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
846 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
847 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
848 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
849 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
850 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
851 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
855 ctxt->ops->put_fpu(ctxt);
858 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
861 ctxt->ops->get_fpu(ctxt);
863 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
864 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
865 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
866 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
867 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
868 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
869 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
870 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
872 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
873 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
874 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
875 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
876 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
877 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
878 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
879 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
883 ctxt->ops->put_fpu(ctxt);
886 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
890 unsigned reg = ctxt->modrm_reg;
891 int highbyte_regs = ctxt->rex_prefix == 0;
893 if (!(ctxt->d & ModRM))
894 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
900 read_sse_reg(ctxt, &op->vec_val, reg);
905 if ((ctxt->d & ByteOp) && !inhibit_bytereg) {
906 op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
909 op->addr.reg = decode_register(reg, ctxt->regs, 0);
910 op->bytes = ctxt->op_bytes;
912 fetch_register_operand(op);
913 op->orig_val = op->val;
916 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
920 int index_reg = 0, base_reg = 0, scale;
921 int rc = X86EMUL_CONTINUE;
924 if (ctxt->rex_prefix) {
925 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
926 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
927 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
930 ctxt->modrm = insn_fetch(u8, ctxt);
931 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
932 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
933 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
934 ctxt->modrm_seg = VCPU_SREG_DS;
936 if (ctxt->modrm_mod == 3) {
938 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
939 op->addr.reg = decode_register(ctxt->modrm_rm,
940 ctxt->regs, ctxt->d & ByteOp);
944 op->addr.xmm = ctxt->modrm_rm;
945 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
948 fetch_register_operand(op);
954 if (ctxt->ad_bytes == 2) {
955 unsigned bx = ctxt->regs[VCPU_REGS_RBX];
956 unsigned bp = ctxt->regs[VCPU_REGS_RBP];
957 unsigned si = ctxt->regs[VCPU_REGS_RSI];
958 unsigned di = ctxt->regs[VCPU_REGS_RDI];
960 /* 16-bit ModR/M decode. */
961 switch (ctxt->modrm_mod) {
963 if (ctxt->modrm_rm == 6)
964 modrm_ea += insn_fetch(u16, ctxt);
967 modrm_ea += insn_fetch(s8, ctxt);
970 modrm_ea += insn_fetch(u16, ctxt);
973 switch (ctxt->modrm_rm) {
993 if (ctxt->modrm_mod != 0)
1000 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1001 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1002 ctxt->modrm_seg = VCPU_SREG_SS;
1003 modrm_ea = (u16)modrm_ea;
1005 /* 32/64-bit ModR/M decode. */
1006 if ((ctxt->modrm_rm & 7) == 4) {
1007 sib = insn_fetch(u8, ctxt);
1008 index_reg |= (sib >> 3) & 7;
1009 base_reg |= sib & 7;
1012 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1013 modrm_ea += insn_fetch(s32, ctxt);
1015 modrm_ea += ctxt->regs[base_reg];
1017 modrm_ea += ctxt->regs[index_reg] << scale;
1018 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1019 if (ctxt->mode == X86EMUL_MODE_PROT64)
1020 ctxt->rip_relative = 1;
1022 modrm_ea += ctxt->regs[ctxt->modrm_rm];
1023 switch (ctxt->modrm_mod) {
1025 if (ctxt->modrm_rm == 5)
1026 modrm_ea += insn_fetch(s32, ctxt);
1029 modrm_ea += insn_fetch(s8, ctxt);
1032 modrm_ea += insn_fetch(s32, ctxt);
1036 op->addr.mem.ea = modrm_ea;
1041 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1044 int rc = X86EMUL_CONTINUE;
1047 switch (ctxt->ad_bytes) {
1049 op->addr.mem.ea = insn_fetch(u16, ctxt);
1052 op->addr.mem.ea = insn_fetch(u32, ctxt);
1055 op->addr.mem.ea = insn_fetch(u64, ctxt);
1062 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1066 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1067 mask = ~(ctxt->dst.bytes * 8 - 1);
1069 if (ctxt->src.bytes == 2)
1070 sv = (s16)ctxt->src.val & (s16)mask;
1071 else if (ctxt->src.bytes == 4)
1072 sv = (s32)ctxt->src.val & (s32)mask;
1074 ctxt->dst.addr.mem.ea += (sv >> 3);
1077 /* only subword offset */
1078 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1081 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1082 unsigned long addr, void *dest, unsigned size)
1085 struct read_cache *mc = &ctxt->mem_read;
1088 int n = min(size, 8u);
1090 if (mc->pos < mc->end)
1093 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1095 if (rc != X86EMUL_CONTINUE)
1100 memcpy(dest, mc->data + mc->pos, n);
1105 return X86EMUL_CONTINUE;
1108 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1109 struct segmented_address addr,
1116 rc = linearize(ctxt, addr, size, false, &linear);
1117 if (rc != X86EMUL_CONTINUE)
1119 return read_emulated(ctxt, linear, data, size);
1122 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1123 struct segmented_address addr,
1130 rc = linearize(ctxt, addr, size, true, &linear);
1131 if (rc != X86EMUL_CONTINUE)
1133 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1137 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1138 struct segmented_address addr,
1139 const void *orig_data, const void *data,
1145 rc = linearize(ctxt, addr, size, true, &linear);
1146 if (rc != X86EMUL_CONTINUE)
1148 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1149 size, &ctxt->exception);
1152 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1153 unsigned int size, unsigned short port,
1156 struct read_cache *rc = &ctxt->io_read;
1158 if (rc->pos == rc->end) { /* refill pio read ahead */
1159 unsigned int in_page, n;
1160 unsigned int count = ctxt->rep_prefix ?
1161 address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
1162 in_page = (ctxt->eflags & EFLG_DF) ?
1163 offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
1164 PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
1165 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1169 rc->pos = rc->end = 0;
1170 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1175 memcpy(dest, rc->data + rc->pos, size);
1180 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1181 u16 selector, struct desc_ptr *dt)
1183 struct x86_emulate_ops *ops = ctxt->ops;
1185 if (selector & 1 << 2) {
1186 struct desc_struct desc;
1189 memset (dt, 0, sizeof *dt);
1190 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1193 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1194 dt->address = get_desc_base(&desc);
1196 ops->get_gdt(ctxt, dt);
1199 /* allowed just for 8 bytes segments */
1200 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1201 u16 selector, struct desc_struct *desc)
1204 u16 index = selector >> 3;
1207 get_descriptor_table_ptr(ctxt, selector, &dt);
1209 if (dt.size < index * 8 + 7)
1210 return emulate_gp(ctxt, selector & 0xfffc);
1212 addr = dt.address + index * 8;
1213 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1217 /* allowed just for 8 bytes segments */
1218 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1219 u16 selector, struct desc_struct *desc)
1222 u16 index = selector >> 3;
1225 get_descriptor_table_ptr(ctxt, selector, &dt);
1227 if (dt.size < index * 8 + 7)
1228 return emulate_gp(ctxt, selector & 0xfffc);
1230 addr = dt.address + index * 8;
1231 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1235 /* Does not support long mode */
1236 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1237 u16 selector, int seg)
1239 struct desc_struct seg_desc;
1241 unsigned err_vec = GP_VECTOR;
1243 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1246 memset(&seg_desc, 0, sizeof seg_desc);
1248 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1249 || ctxt->mode == X86EMUL_MODE_REAL) {
1250 /* set real mode segment descriptor */
1251 set_desc_base(&seg_desc, selector << 4);
1252 set_desc_limit(&seg_desc, 0xffff);
1259 /* NULL selector is not valid for TR, CS and SS */
1260 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1264 /* TR should be in GDT only */
1265 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1268 if (null_selector) /* for NULL selector skip all following checks */
1271 ret = read_segment_descriptor(ctxt, selector, &seg_desc);
1272 if (ret != X86EMUL_CONTINUE)
1275 err_code = selector & 0xfffc;
1276 err_vec = GP_VECTOR;
1278 /* can't load system descriptor into segment selecor */
1279 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1283 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1289 cpl = ctxt->ops->cpl(ctxt);
1294 * segment is not a writable data segment or segment
1295 * selector's RPL != CPL or segment selector's RPL != CPL
1297 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1301 if (!(seg_desc.type & 8))
1304 if (seg_desc.type & 4) {
1310 if (rpl > cpl || dpl != cpl)
1313 /* CS(RPL) <- CPL */
1314 selector = (selector & 0xfffc) | cpl;
1317 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1320 case VCPU_SREG_LDTR:
1321 if (seg_desc.s || seg_desc.type != 2)
1324 default: /* DS, ES, FS, or GS */
1326 * segment is not a data or readable code segment or
1327 * ((segment is a data or nonconforming code segment)
1328 * and (both RPL and CPL > DPL))
1330 if ((seg_desc.type & 0xa) == 0x8 ||
1331 (((seg_desc.type & 0xc) != 0xc) &&
1332 (rpl > dpl && cpl > dpl)))
1338 /* mark segment as accessed */
1340 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1341 if (ret != X86EMUL_CONTINUE)
1345 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1346 return X86EMUL_CONTINUE;
1348 emulate_exception(ctxt, err_vec, err_code, true);
1349 return X86EMUL_PROPAGATE_FAULT;
1352 static void write_register_operand(struct operand *op)
1354 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1355 switch (op->bytes) {
1357 *(u8 *)op->addr.reg = (u8)op->val;
1360 *(u16 *)op->addr.reg = (u16)op->val;
1363 *op->addr.reg = (u32)op->val;
1364 break; /* 64b: zero-extend */
1366 *op->addr.reg = op->val;
1371 static int writeback(struct x86_emulate_ctxt *ctxt)
1375 switch (ctxt->dst.type) {
1377 write_register_operand(&ctxt->dst);
1380 if (ctxt->lock_prefix)
1381 rc = segmented_cmpxchg(ctxt,
1383 &ctxt->dst.orig_val,
1387 rc = segmented_write(ctxt,
1391 if (rc != X86EMUL_CONTINUE)
1395 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1403 return X86EMUL_CONTINUE;
1406 static int em_push(struct x86_emulate_ctxt *ctxt)
1408 struct segmented_address addr;
1410 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], -ctxt->op_bytes);
1411 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1412 addr.seg = VCPU_SREG_SS;
1414 /* Disable writeback. */
1415 ctxt->dst.type = OP_NONE;
1416 return segmented_write(ctxt, addr, &ctxt->src.val, ctxt->op_bytes);
1419 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1420 void *dest, int len)
1423 struct segmented_address addr;
1425 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1426 addr.seg = VCPU_SREG_SS;
1427 rc = segmented_read(ctxt, addr, dest, len);
1428 if (rc != X86EMUL_CONTINUE)
1431 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], len);
1435 static int em_pop(struct x86_emulate_ctxt *ctxt)
1437 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1440 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1441 void *dest, int len)
1444 unsigned long val, change_mask;
1445 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1446 int cpl = ctxt->ops->cpl(ctxt);
1448 rc = emulate_pop(ctxt, &val, len);
1449 if (rc != X86EMUL_CONTINUE)
1452 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1453 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1455 switch(ctxt->mode) {
1456 case X86EMUL_MODE_PROT64:
1457 case X86EMUL_MODE_PROT32:
1458 case X86EMUL_MODE_PROT16:
1460 change_mask |= EFLG_IOPL;
1462 change_mask |= EFLG_IF;
1464 case X86EMUL_MODE_VM86:
1466 return emulate_gp(ctxt, 0);
1467 change_mask |= EFLG_IF;
1469 default: /* real mode */
1470 change_mask |= (EFLG_IOPL | EFLG_IF);
1474 *(unsigned long *)dest =
1475 (ctxt->eflags & ~change_mask) | (val & change_mask);
1480 static int em_popf(struct x86_emulate_ctxt *ctxt)
1482 ctxt->dst.type = OP_REG;
1483 ctxt->dst.addr.reg = &ctxt->eflags;
1484 ctxt->dst.bytes = ctxt->op_bytes;
1485 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1488 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1490 int seg = ctxt->src2.val;
1492 ctxt->src.val = get_segment_selector(ctxt, seg);
1494 return em_push(ctxt);
1497 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1499 int seg = ctxt->src2.val;
1500 unsigned long selector;
1503 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1504 if (rc != X86EMUL_CONTINUE)
1507 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1511 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1513 unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
1514 int rc = X86EMUL_CONTINUE;
1515 int reg = VCPU_REGS_RAX;
1517 while (reg <= VCPU_REGS_RDI) {
1518 (reg == VCPU_REGS_RSP) ?
1519 (ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
1522 if (rc != X86EMUL_CONTINUE)
1531 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1533 ctxt->src.val = (unsigned long)ctxt->eflags;
1534 return em_push(ctxt);
1537 static int em_popa(struct x86_emulate_ctxt *ctxt)
1539 int rc = X86EMUL_CONTINUE;
1540 int reg = VCPU_REGS_RDI;
1542 while (reg >= VCPU_REGS_RAX) {
1543 if (reg == VCPU_REGS_RSP) {
1544 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP],
1549 rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
1550 if (rc != X86EMUL_CONTINUE)
1557 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1559 struct x86_emulate_ops *ops = ctxt->ops;
1566 /* TODO: Add limit checks */
1567 ctxt->src.val = ctxt->eflags;
1569 if (rc != X86EMUL_CONTINUE)
1572 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1574 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1576 if (rc != X86EMUL_CONTINUE)
1579 ctxt->src.val = ctxt->_eip;
1581 if (rc != X86EMUL_CONTINUE)
1584 ops->get_idt(ctxt, &dt);
1586 eip_addr = dt.address + (irq << 2);
1587 cs_addr = dt.address + (irq << 2) + 2;
1589 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1590 if (rc != X86EMUL_CONTINUE)
1593 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1594 if (rc != X86EMUL_CONTINUE)
1597 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1598 if (rc != X86EMUL_CONTINUE)
1606 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1608 switch(ctxt->mode) {
1609 case X86EMUL_MODE_REAL:
1610 return emulate_int_real(ctxt, irq);
1611 case X86EMUL_MODE_VM86:
1612 case X86EMUL_MODE_PROT16:
1613 case X86EMUL_MODE_PROT32:
1614 case X86EMUL_MODE_PROT64:
1616 /* Protected mode interrupts unimplemented yet */
1617 return X86EMUL_UNHANDLEABLE;
1621 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1623 int rc = X86EMUL_CONTINUE;
1624 unsigned long temp_eip = 0;
1625 unsigned long temp_eflags = 0;
1626 unsigned long cs = 0;
1627 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1628 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1629 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1630 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1632 /* TODO: Add stack limit check */
1634 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1636 if (rc != X86EMUL_CONTINUE)
1639 if (temp_eip & ~0xffff)
1640 return emulate_gp(ctxt, 0);
1642 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1644 if (rc != X86EMUL_CONTINUE)
1647 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1649 if (rc != X86EMUL_CONTINUE)
1652 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1654 if (rc != X86EMUL_CONTINUE)
1657 ctxt->_eip = temp_eip;
1660 if (ctxt->op_bytes == 4)
1661 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1662 else if (ctxt->op_bytes == 2) {
1663 ctxt->eflags &= ~0xffff;
1664 ctxt->eflags |= temp_eflags;
1667 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1668 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1673 static int em_iret(struct x86_emulate_ctxt *ctxt)
1675 switch(ctxt->mode) {
1676 case X86EMUL_MODE_REAL:
1677 return emulate_iret_real(ctxt);
1678 case X86EMUL_MODE_VM86:
1679 case X86EMUL_MODE_PROT16:
1680 case X86EMUL_MODE_PROT32:
1681 case X86EMUL_MODE_PROT64:
1683 /* iret from protected mode unimplemented yet */
1684 return X86EMUL_UNHANDLEABLE;
1688 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1693 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1695 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1696 if (rc != X86EMUL_CONTINUE)
1700 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1701 return X86EMUL_CONTINUE;
1704 static int em_grp1a(struct x86_emulate_ctxt *ctxt)
1706 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->dst.bytes);
1709 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1711 switch (ctxt->modrm_reg) {
1713 emulate_2op_SrcB(ctxt, "rol");
1716 emulate_2op_SrcB(ctxt, "ror");
1719 emulate_2op_SrcB(ctxt, "rcl");
1722 emulate_2op_SrcB(ctxt, "rcr");
1724 case 4: /* sal/shl */
1725 case 6: /* sal/shl */
1726 emulate_2op_SrcB(ctxt, "sal");
1729 emulate_2op_SrcB(ctxt, "shr");
1732 emulate_2op_SrcB(ctxt, "sar");
1735 return X86EMUL_CONTINUE;
1738 static int em_not(struct x86_emulate_ctxt *ctxt)
1740 ctxt->dst.val = ~ctxt->dst.val;
1741 return X86EMUL_CONTINUE;
1744 static int em_neg(struct x86_emulate_ctxt *ctxt)
1746 emulate_1op(ctxt, "neg");
1747 return X86EMUL_CONTINUE;
1750 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
1754 emulate_1op_rax_rdx(ctxt, "mul", ex);
1755 return X86EMUL_CONTINUE;
1758 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
1762 emulate_1op_rax_rdx(ctxt, "imul", ex);
1763 return X86EMUL_CONTINUE;
1766 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1770 emulate_1op_rax_rdx(ctxt, "div", de);
1772 return emulate_de(ctxt);
1773 return X86EMUL_CONTINUE;
1776 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
1780 emulate_1op_rax_rdx(ctxt, "idiv", de);
1782 return emulate_de(ctxt);
1783 return X86EMUL_CONTINUE;
1786 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1788 int rc = X86EMUL_CONTINUE;
1790 switch (ctxt->modrm_reg) {
1792 emulate_1op(ctxt, "inc");
1795 emulate_1op(ctxt, "dec");
1797 case 2: /* call near abs */ {
1799 old_eip = ctxt->_eip;
1800 rc = assign_eip_near(ctxt, ctxt->src.val);
1801 if (rc != X86EMUL_CONTINUE)
1803 ctxt->src.val = old_eip;
1807 case 4: /* jmp abs */
1808 rc = assign_eip_near(ctxt, ctxt->src.val);
1810 case 5: /* jmp far */
1811 rc = em_jmp_far(ctxt);
1820 static int em_grp9(struct x86_emulate_ctxt *ctxt)
1822 u64 old = ctxt->dst.orig_val64;
1824 if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
1825 ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
1826 ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1827 ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1828 ctxt->eflags &= ~EFLG_ZF;
1830 ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
1831 (u32) ctxt->regs[VCPU_REGS_RBX];
1833 ctxt->eflags |= EFLG_ZF;
1835 return X86EMUL_CONTINUE;
1838 static int em_ret(struct x86_emulate_ctxt *ctxt)
1843 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
1844 if (rc != X86EMUL_CONTINUE)
1847 return assign_eip_near(ctxt, eip);
1850 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
1854 int cpl = ctxt->ops->cpl(ctxt);
1856 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
1857 if (rc != X86EMUL_CONTINUE)
1859 if (ctxt->op_bytes == 4)
1860 ctxt->_eip = (u32)ctxt->_eip;
1861 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1862 if (rc != X86EMUL_CONTINUE)
1864 /* Outer-privilege level return is not implemented */
1865 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
1866 return X86EMUL_UNHANDLEABLE;
1867 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1871 static int em_lseg(struct x86_emulate_ctxt *ctxt)
1873 int seg = ctxt->src2.val;
1877 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1879 rc = load_segment_descriptor(ctxt, sel, seg);
1880 if (rc != X86EMUL_CONTINUE)
1883 ctxt->dst.val = ctxt->src.val;
1888 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1889 struct desc_struct *cs, struct desc_struct *ss)
1893 memset(cs, 0, sizeof(struct desc_struct));
1894 ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1895 memset(ss, 0, sizeof(struct desc_struct));
1897 cs->l = 0; /* will be adjusted later */
1898 set_desc_base(cs, 0); /* flat segment */
1899 cs->g = 1; /* 4kb granularity */
1900 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1901 cs->type = 0x0b; /* Read, Execute, Accessed */
1903 cs->dpl = 0; /* will be adjusted later */
1907 set_desc_base(ss, 0); /* flat segment */
1908 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1909 ss->g = 1; /* 4kb granularity */
1911 ss->type = 0x03; /* Read/Write, Accessed */
1912 ss->d = 1; /* 32bit stack segment */
1917 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
1919 struct x86_emulate_ops *ops = ctxt->ops;
1920 u32 eax, ebx, ecx, edx;
1923 * syscall should always be enabled in longmode - so only become
1924 * vendor specific (cpuid) if other modes are active...
1926 if (ctxt->mode == X86EMUL_MODE_PROT64)
1931 if (ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx)) {
1933 * Intel ("GenuineIntel")
1934 * remark: Intel CPUs only support "syscall" in 64bit
1935 * longmode. Also an 64bit guest with a
1936 * 32bit compat-app running will #UD !! While this
1937 * behaviour can be fixed (by emulating) into AMD
1938 * response - CPUs of AMD can't behave like Intel.
1940 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
1941 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
1942 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
1945 /* AMD ("AuthenticAMD") */
1946 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
1947 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
1948 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
1951 /* AMD ("AMDisbetter!") */
1952 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
1953 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
1954 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
1958 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
1962 static int em_syscall(struct x86_emulate_ctxt *ctxt)
1964 struct x86_emulate_ops *ops = ctxt->ops;
1965 struct desc_struct cs, ss;
1970 /* syscall is not available in real mode */
1971 if (ctxt->mode == X86EMUL_MODE_REAL ||
1972 ctxt->mode == X86EMUL_MODE_VM86)
1973 return emulate_ud(ctxt);
1975 if (!(em_syscall_is_enabled(ctxt)))
1976 return emulate_ud(ctxt);
1978 ops->get_msr(ctxt, MSR_EFER, &efer);
1979 setup_syscalls_segments(ctxt, &cs, &ss);
1981 if (!(efer & EFER_SCE))
1982 return emulate_ud(ctxt);
1984 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1986 cs_sel = (u16)(msr_data & 0xfffc);
1987 ss_sel = (u16)(msr_data + 8);
1989 if (efer & EFER_LMA) {
1993 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1994 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1996 ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
1997 if (efer & EFER_LMA) {
1998 #ifdef CONFIG_X86_64
1999 ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
2002 ctxt->mode == X86EMUL_MODE_PROT64 ?
2003 MSR_LSTAR : MSR_CSTAR, &msr_data);
2004 ctxt->_eip = msr_data;
2006 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2007 ctxt->eflags &= ~(msr_data | EFLG_RF);
2011 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2012 ctxt->_eip = (u32)msr_data;
2014 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2017 return X86EMUL_CONTINUE;
2020 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2022 struct x86_emulate_ops *ops = ctxt->ops;
2023 struct desc_struct cs, ss;
2028 ops->get_msr(ctxt, MSR_EFER, &efer);
2029 /* inject #GP if in real mode */
2030 if (ctxt->mode == X86EMUL_MODE_REAL)
2031 return emulate_gp(ctxt, 0);
2033 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2034 * Therefore, we inject an #UD.
2036 if (ctxt->mode == X86EMUL_MODE_PROT64)
2037 return emulate_ud(ctxt);
2039 setup_syscalls_segments(ctxt, &cs, &ss);
2041 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2042 switch (ctxt->mode) {
2043 case X86EMUL_MODE_PROT32:
2044 if ((msr_data & 0xfffc) == 0x0)
2045 return emulate_gp(ctxt, 0);
2047 case X86EMUL_MODE_PROT64:
2048 if (msr_data == 0x0)
2049 return emulate_gp(ctxt, 0);
2053 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2054 cs_sel = (u16)msr_data;
2055 cs_sel &= ~SELECTOR_RPL_MASK;
2056 ss_sel = cs_sel + 8;
2057 ss_sel &= ~SELECTOR_RPL_MASK;
2058 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2063 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2064 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2066 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2067 ctxt->_eip = msr_data;
2069 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2070 ctxt->regs[VCPU_REGS_RSP] = msr_data;
2072 return X86EMUL_CONTINUE;
2075 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2077 struct x86_emulate_ops *ops = ctxt->ops;
2078 struct desc_struct cs, ss;
2079 u64 msr_data, rcx, rdx;
2081 u16 cs_sel = 0, ss_sel = 0;
2083 /* inject #GP if in real mode or Virtual 8086 mode */
2084 if (ctxt->mode == X86EMUL_MODE_REAL ||
2085 ctxt->mode == X86EMUL_MODE_VM86)
2086 return emulate_gp(ctxt, 0);
2088 setup_syscalls_segments(ctxt, &cs, &ss);
2090 if ((ctxt->rex_prefix & 0x8) != 0x0)
2091 usermode = X86EMUL_MODE_PROT64;
2093 usermode = X86EMUL_MODE_PROT32;
2095 rcx = ctxt->regs[VCPU_REGS_RCX];
2096 rdx = ctxt->regs[VCPU_REGS_RDX];
2100 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2102 case X86EMUL_MODE_PROT32:
2103 cs_sel = (u16)(msr_data + 16);
2104 if ((msr_data & 0xfffc) == 0x0)
2105 return emulate_gp(ctxt, 0);
2106 ss_sel = (u16)(msr_data + 24);
2108 case X86EMUL_MODE_PROT64:
2109 cs_sel = (u16)(msr_data + 32);
2110 if (msr_data == 0x0)
2111 return emulate_gp(ctxt, 0);
2112 ss_sel = cs_sel + 8;
2115 if (is_noncanonical_address(rcx) ||
2116 is_noncanonical_address(rdx))
2117 return emulate_gp(ctxt, 0);
2120 cs_sel |= SELECTOR_RPL_MASK;
2121 ss_sel |= SELECTOR_RPL_MASK;
2123 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2124 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2127 ctxt->regs[VCPU_REGS_RSP] = rcx;
2129 return X86EMUL_CONTINUE;
2132 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2135 if (ctxt->mode == X86EMUL_MODE_REAL)
2137 if (ctxt->mode == X86EMUL_MODE_VM86)
2139 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2140 return ctxt->ops->cpl(ctxt) > iopl;
2143 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2146 struct x86_emulate_ops *ops = ctxt->ops;
2147 struct desc_struct tr_seg;
2150 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2151 unsigned mask = (1 << len) - 1;
2154 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2157 if (desc_limit_scaled(&tr_seg) < 103)
2159 base = get_desc_base(&tr_seg);
2160 #ifdef CONFIG_X86_64
2161 base |= ((u64)base3) << 32;
2163 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2164 if (r != X86EMUL_CONTINUE)
2166 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2168 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2169 if (r != X86EMUL_CONTINUE)
2171 if ((perm >> bit_idx) & mask)
2176 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2182 if (emulator_bad_iopl(ctxt))
2183 if (!emulator_io_port_access_allowed(ctxt, port, len))
2186 ctxt->perm_ok = true;
2191 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2192 struct tss_segment_16 *tss)
2194 tss->ip = ctxt->_eip;
2195 tss->flag = ctxt->eflags;
2196 tss->ax = ctxt->regs[VCPU_REGS_RAX];
2197 tss->cx = ctxt->regs[VCPU_REGS_RCX];
2198 tss->dx = ctxt->regs[VCPU_REGS_RDX];
2199 tss->bx = ctxt->regs[VCPU_REGS_RBX];
2200 tss->sp = ctxt->regs[VCPU_REGS_RSP];
2201 tss->bp = ctxt->regs[VCPU_REGS_RBP];
2202 tss->si = ctxt->regs[VCPU_REGS_RSI];
2203 tss->di = ctxt->regs[VCPU_REGS_RDI];
2205 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2206 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2207 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2208 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2209 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2212 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2213 struct tss_segment_16 *tss)
2217 ctxt->_eip = tss->ip;
2218 ctxt->eflags = tss->flag | 2;
2219 ctxt->regs[VCPU_REGS_RAX] = tss->ax;
2220 ctxt->regs[VCPU_REGS_RCX] = tss->cx;
2221 ctxt->regs[VCPU_REGS_RDX] = tss->dx;
2222 ctxt->regs[VCPU_REGS_RBX] = tss->bx;
2223 ctxt->regs[VCPU_REGS_RSP] = tss->sp;
2224 ctxt->regs[VCPU_REGS_RBP] = tss->bp;
2225 ctxt->regs[VCPU_REGS_RSI] = tss->si;
2226 ctxt->regs[VCPU_REGS_RDI] = tss->di;
2229 * SDM says that segment selectors are loaded before segment
2232 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2233 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2234 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2235 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2236 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2239 * Now load segment descriptors. If fault happenes at this stage
2240 * it is handled in a context of new task
2242 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2243 if (ret != X86EMUL_CONTINUE)
2245 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2246 if (ret != X86EMUL_CONTINUE)
2248 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2249 if (ret != X86EMUL_CONTINUE)
2251 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2252 if (ret != X86EMUL_CONTINUE)
2254 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2255 if (ret != X86EMUL_CONTINUE)
2258 return X86EMUL_CONTINUE;
2261 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2262 u16 tss_selector, u16 old_tss_sel,
2263 ulong old_tss_base, struct desc_struct *new_desc)
2265 struct x86_emulate_ops *ops = ctxt->ops;
2266 struct tss_segment_16 tss_seg;
2268 u32 new_tss_base = get_desc_base(new_desc);
2270 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2272 if (ret != X86EMUL_CONTINUE)
2273 /* FIXME: need to provide precise fault address */
2276 save_state_to_tss16(ctxt, &tss_seg);
2278 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2280 if (ret != X86EMUL_CONTINUE)
2281 /* FIXME: need to provide precise fault address */
2284 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2286 if (ret != X86EMUL_CONTINUE)
2287 /* FIXME: need to provide precise fault address */
2290 if (old_tss_sel != 0xffff) {
2291 tss_seg.prev_task_link = old_tss_sel;
2293 ret = ops->write_std(ctxt, new_tss_base,
2294 &tss_seg.prev_task_link,
2295 sizeof tss_seg.prev_task_link,
2297 if (ret != X86EMUL_CONTINUE)
2298 /* FIXME: need to provide precise fault address */
2302 return load_state_from_tss16(ctxt, &tss_seg);
2305 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2306 struct tss_segment_32 *tss)
2308 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2309 tss->eip = ctxt->_eip;
2310 tss->eflags = ctxt->eflags;
2311 tss->eax = ctxt->regs[VCPU_REGS_RAX];
2312 tss->ecx = ctxt->regs[VCPU_REGS_RCX];
2313 tss->edx = ctxt->regs[VCPU_REGS_RDX];
2314 tss->ebx = ctxt->regs[VCPU_REGS_RBX];
2315 tss->esp = ctxt->regs[VCPU_REGS_RSP];
2316 tss->ebp = ctxt->regs[VCPU_REGS_RBP];
2317 tss->esi = ctxt->regs[VCPU_REGS_RSI];
2318 tss->edi = ctxt->regs[VCPU_REGS_RDI];
2320 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2321 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2322 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2323 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2324 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2325 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2326 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2329 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2330 struct tss_segment_32 *tss)
2334 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2335 return emulate_gp(ctxt, 0);
2336 ctxt->_eip = tss->eip;
2337 ctxt->eflags = tss->eflags | 2;
2338 ctxt->regs[VCPU_REGS_RAX] = tss->eax;
2339 ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
2340 ctxt->regs[VCPU_REGS_RDX] = tss->edx;
2341 ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
2342 ctxt->regs[VCPU_REGS_RSP] = tss->esp;
2343 ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
2344 ctxt->regs[VCPU_REGS_RSI] = tss->esi;
2345 ctxt->regs[VCPU_REGS_RDI] = tss->edi;
2348 * SDM says that segment selectors are loaded before segment
2351 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2352 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2353 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2354 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2355 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2356 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2357 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2360 * Now load segment descriptors. If fault happenes at this stage
2361 * it is handled in a context of new task
2363 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2364 if (ret != X86EMUL_CONTINUE)
2366 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2367 if (ret != X86EMUL_CONTINUE)
2369 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2370 if (ret != X86EMUL_CONTINUE)
2372 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2373 if (ret != X86EMUL_CONTINUE)
2375 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2376 if (ret != X86EMUL_CONTINUE)
2378 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2379 if (ret != X86EMUL_CONTINUE)
2381 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2382 if (ret != X86EMUL_CONTINUE)
2385 return X86EMUL_CONTINUE;
2388 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2389 u16 tss_selector, u16 old_tss_sel,
2390 ulong old_tss_base, struct desc_struct *new_desc)
2392 struct x86_emulate_ops *ops = ctxt->ops;
2393 struct tss_segment_32 tss_seg;
2395 u32 new_tss_base = get_desc_base(new_desc);
2397 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2399 if (ret != X86EMUL_CONTINUE)
2400 /* FIXME: need to provide precise fault address */
2403 save_state_to_tss32(ctxt, &tss_seg);
2405 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2407 if (ret != X86EMUL_CONTINUE)
2408 /* FIXME: need to provide precise fault address */
2411 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2413 if (ret != X86EMUL_CONTINUE)
2414 /* FIXME: need to provide precise fault address */
2417 if (old_tss_sel != 0xffff) {
2418 tss_seg.prev_task_link = old_tss_sel;
2420 ret = ops->write_std(ctxt, new_tss_base,
2421 &tss_seg.prev_task_link,
2422 sizeof tss_seg.prev_task_link,
2424 if (ret != X86EMUL_CONTINUE)
2425 /* FIXME: need to provide precise fault address */
2429 return load_state_from_tss32(ctxt, &tss_seg);
2432 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2433 u16 tss_selector, int reason,
2434 bool has_error_code, u32 error_code)
2436 struct x86_emulate_ops *ops = ctxt->ops;
2437 struct desc_struct curr_tss_desc, next_tss_desc;
2439 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2440 ulong old_tss_base =
2441 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2444 /* FIXME: old_tss_base == ~0 ? */
2446 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2447 if (ret != X86EMUL_CONTINUE)
2449 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2450 if (ret != X86EMUL_CONTINUE)
2453 /* FIXME: check that next_tss_desc is tss */
2455 if (reason != TASK_SWITCH_IRET) {
2456 if ((tss_selector & 3) > next_tss_desc.dpl ||
2457 ops->cpl(ctxt) > next_tss_desc.dpl)
2458 return emulate_gp(ctxt, 0);
2461 desc_limit = desc_limit_scaled(&next_tss_desc);
2462 if (!next_tss_desc.p ||
2463 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2464 desc_limit < 0x2b)) {
2465 emulate_ts(ctxt, tss_selector & 0xfffc);
2466 return X86EMUL_PROPAGATE_FAULT;
2469 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2470 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2471 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2474 if (reason == TASK_SWITCH_IRET)
2475 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2477 /* set back link to prev task only if NT bit is set in eflags
2478 note that old_tss_sel is not used afetr this point */
2479 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2480 old_tss_sel = 0xffff;
2482 if (next_tss_desc.type & 8)
2483 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2484 old_tss_base, &next_tss_desc);
2486 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2487 old_tss_base, &next_tss_desc);
2488 if (ret != X86EMUL_CONTINUE)
2491 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2492 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2494 if (reason != TASK_SWITCH_IRET) {
2495 next_tss_desc.type |= (1 << 1); /* set busy flag */
2496 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2499 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2500 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2502 if (has_error_code) {
2503 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2504 ctxt->lock_prefix = 0;
2505 ctxt->src.val = (unsigned long) error_code;
2506 ret = em_push(ctxt);
2512 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2513 u16 tss_selector, int reason,
2514 bool has_error_code, u32 error_code)
2518 ctxt->_eip = ctxt->eip;
2519 ctxt->dst.type = OP_NONE;
2521 rc = emulator_do_task_switch(ctxt, tss_selector, reason,
2522 has_error_code, error_code);
2524 if (rc == X86EMUL_CONTINUE)
2525 ctxt->eip = ctxt->_eip;
2527 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2530 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2531 int reg, struct operand *op)
2533 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2535 register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
2536 op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
2537 op->addr.mem.seg = seg;
2540 static int em_das(struct x86_emulate_ctxt *ctxt)
2543 bool af, cf, old_cf;
2545 cf = ctxt->eflags & X86_EFLAGS_CF;
2551 af = ctxt->eflags & X86_EFLAGS_AF;
2552 if ((al & 0x0f) > 9 || af) {
2554 cf = old_cf | (al >= 250);
2559 if (old_al > 0x99 || old_cf) {
2565 /* Set PF, ZF, SF */
2566 ctxt->src.type = OP_IMM;
2568 ctxt->src.bytes = 1;
2569 emulate_2op_SrcV(ctxt, "or");
2570 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2572 ctxt->eflags |= X86_EFLAGS_CF;
2574 ctxt->eflags |= X86_EFLAGS_AF;
2575 return X86EMUL_CONTINUE;
2578 static int em_call(struct x86_emulate_ctxt *ctxt)
2581 long rel = ctxt->src.val;
2583 ctxt->src.val = (unsigned long)ctxt->_eip;
2584 rc = jmp_rel(ctxt, rel);
2585 if (rc != X86EMUL_CONTINUE)
2587 return em_push(ctxt);
2590 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2596 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2597 old_eip = ctxt->_eip;
2599 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2600 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2601 return X86EMUL_CONTINUE;
2604 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2606 ctxt->src.val = old_cs;
2608 if (rc != X86EMUL_CONTINUE)
2611 ctxt->src.val = old_eip;
2612 return em_push(ctxt);
2615 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2620 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2621 if (rc != X86EMUL_CONTINUE)
2623 rc = assign_eip_near(ctxt, eip);
2624 if (rc != X86EMUL_CONTINUE)
2626 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], ctxt->src.val);
2627 return X86EMUL_CONTINUE;
2630 static int em_add(struct x86_emulate_ctxt *ctxt)
2632 emulate_2op_SrcV(ctxt, "add");
2633 return X86EMUL_CONTINUE;
2636 static int em_or(struct x86_emulate_ctxt *ctxt)
2638 emulate_2op_SrcV(ctxt, "or");
2639 return X86EMUL_CONTINUE;
2642 static int em_adc(struct x86_emulate_ctxt *ctxt)
2644 emulate_2op_SrcV(ctxt, "adc");
2645 return X86EMUL_CONTINUE;
2648 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2650 emulate_2op_SrcV(ctxt, "sbb");
2651 return X86EMUL_CONTINUE;
2654 static int em_and(struct x86_emulate_ctxt *ctxt)
2656 emulate_2op_SrcV(ctxt, "and");
2657 return X86EMUL_CONTINUE;
2660 static int em_sub(struct x86_emulate_ctxt *ctxt)
2662 emulate_2op_SrcV(ctxt, "sub");
2663 return X86EMUL_CONTINUE;
2666 static int em_xor(struct x86_emulate_ctxt *ctxt)
2668 emulate_2op_SrcV(ctxt, "xor");
2669 return X86EMUL_CONTINUE;
2672 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2674 emulate_2op_SrcV(ctxt, "cmp");
2675 /* Disable writeback. */
2676 ctxt->dst.type = OP_NONE;
2677 return X86EMUL_CONTINUE;
2680 static int em_test(struct x86_emulate_ctxt *ctxt)
2682 emulate_2op_SrcV(ctxt, "test");
2683 /* Disable writeback. */
2684 ctxt->dst.type = OP_NONE;
2685 return X86EMUL_CONTINUE;
2688 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2690 /* Write back the register source. */
2691 ctxt->src.val = ctxt->dst.val;
2692 write_register_operand(&ctxt->src);
2694 /* Write back the memory destination with implicit LOCK prefix. */
2695 ctxt->dst.val = ctxt->src.orig_val;
2696 ctxt->lock_prefix = 1;
2697 return X86EMUL_CONTINUE;
2700 static int em_imul(struct x86_emulate_ctxt *ctxt)
2702 emulate_2op_SrcV_nobyte(ctxt, "imul");
2703 return X86EMUL_CONTINUE;
2706 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2708 ctxt->dst.val = ctxt->src2.val;
2709 return em_imul(ctxt);
2712 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2714 ctxt->dst.type = OP_REG;
2715 ctxt->dst.bytes = ctxt->src.bytes;
2716 ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
2717 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2719 return X86EMUL_CONTINUE;
2722 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2726 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2727 ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
2728 ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
2729 return X86EMUL_CONTINUE;
2732 static int em_mov(struct x86_emulate_ctxt *ctxt)
2734 ctxt->dst.val = ctxt->src.val;
2735 return X86EMUL_CONTINUE;
2738 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
2740 if (ctxt->modrm_reg > VCPU_SREG_GS)
2741 return emulate_ud(ctxt);
2743 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
2744 return X86EMUL_CONTINUE;
2747 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
2749 u16 sel = ctxt->src.val;
2751 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
2752 return emulate_ud(ctxt);
2754 if (ctxt->modrm_reg == VCPU_SREG_SS)
2755 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2757 /* Disable writeback. */
2758 ctxt->dst.type = OP_NONE;
2759 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
2762 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2764 memcpy(&ctxt->dst.vec_val, &ctxt->src.vec_val, ctxt->op_bytes);
2765 return X86EMUL_CONTINUE;
2768 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2773 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
2774 if (rc == X86EMUL_CONTINUE)
2775 ctxt->ops->invlpg(ctxt, linear);
2776 /* Disable writeback. */
2777 ctxt->dst.type = OP_NONE;
2778 return X86EMUL_CONTINUE;
2781 static int em_clts(struct x86_emulate_ctxt *ctxt)
2785 cr0 = ctxt->ops->get_cr(ctxt, 0);
2787 ctxt->ops->set_cr(ctxt, 0, cr0);
2788 return X86EMUL_CONTINUE;
2791 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
2795 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
2796 return X86EMUL_UNHANDLEABLE;
2798 rc = ctxt->ops->fix_hypercall(ctxt);
2799 if (rc != X86EMUL_CONTINUE)
2802 /* Let the processor re-execute the fixed hypercall */
2803 ctxt->_eip = ctxt->eip;
2804 /* Disable writeback. */
2805 ctxt->dst.type = OP_NONE;
2806 return X86EMUL_CONTINUE;
2809 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
2811 struct desc_ptr desc_ptr;
2814 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2815 &desc_ptr.size, &desc_ptr.address,
2817 if (rc != X86EMUL_CONTINUE)
2819 ctxt->ops->set_gdt(ctxt, &desc_ptr);
2820 /* Disable writeback. */
2821 ctxt->dst.type = OP_NONE;
2822 return X86EMUL_CONTINUE;
2825 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2829 rc = ctxt->ops->fix_hypercall(ctxt);
2831 /* Disable writeback. */
2832 ctxt->dst.type = OP_NONE;
2836 static int em_lidt(struct x86_emulate_ctxt *ctxt)
2838 struct desc_ptr desc_ptr;
2841 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2842 &desc_ptr.size, &desc_ptr.address,
2844 if (rc != X86EMUL_CONTINUE)
2846 ctxt->ops->set_idt(ctxt, &desc_ptr);
2847 /* Disable writeback. */
2848 ctxt->dst.type = OP_NONE;
2849 return X86EMUL_CONTINUE;
2852 static int em_smsw(struct x86_emulate_ctxt *ctxt)
2854 ctxt->dst.bytes = 2;
2855 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
2856 return X86EMUL_CONTINUE;
2859 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
2861 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2862 | (ctxt->src.val & 0x0f));
2863 ctxt->dst.type = OP_NONE;
2864 return X86EMUL_CONTINUE;
2867 static int em_loop(struct x86_emulate_ctxt *ctxt)
2869 int rc = X86EMUL_CONTINUE;
2871 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
2872 if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
2873 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
2874 rc = jmp_rel(ctxt, ctxt->src.val);
2879 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
2881 int rc = X86EMUL_CONTINUE;
2883 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
2884 rc = jmp_rel(ctxt, ctxt->src.val);
2889 static int em_cli(struct x86_emulate_ctxt *ctxt)
2891 if (emulator_bad_iopl(ctxt))
2892 return emulate_gp(ctxt, 0);
2894 ctxt->eflags &= ~X86_EFLAGS_IF;
2895 return X86EMUL_CONTINUE;
2898 static int em_sti(struct x86_emulate_ctxt *ctxt)
2900 if (emulator_bad_iopl(ctxt))
2901 return emulate_gp(ctxt, 0);
2903 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
2904 ctxt->eflags |= X86_EFLAGS_IF;
2905 return X86EMUL_CONTINUE;
2908 static bool valid_cr(int nr)
2920 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2922 if (!valid_cr(ctxt->modrm_reg))
2923 return emulate_ud(ctxt);
2925 return X86EMUL_CONTINUE;
2928 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2930 u64 new_val = ctxt->src.val64;
2931 int cr = ctxt->modrm_reg;
2934 static u64 cr_reserved_bits[] = {
2935 0xffffffff00000000ULL,
2936 0, 0, 0, /* CR3 checked later */
2943 return emulate_ud(ctxt);
2945 if (new_val & cr_reserved_bits[cr])
2946 return emulate_gp(ctxt, 0);
2951 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2952 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2953 return emulate_gp(ctxt, 0);
2955 cr4 = ctxt->ops->get_cr(ctxt, 4);
2956 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2958 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2959 !(cr4 & X86_CR4_PAE))
2960 return emulate_gp(ctxt, 0);
2967 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2968 if (efer & EFER_LMA)
2969 rsvd = CR3_L_MODE_RESERVED_BITS;
2970 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
2971 rsvd = CR3_PAE_RESERVED_BITS;
2972 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
2973 rsvd = CR3_NONPAE_RESERVED_BITS;
2976 return emulate_gp(ctxt, 0);
2983 cr4 = ctxt->ops->get_cr(ctxt, 4);
2984 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2986 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2987 return emulate_gp(ctxt, 0);
2993 return X86EMUL_CONTINUE;
2996 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3000 ctxt->ops->get_dr(ctxt, 7, &dr7);
3002 /* Check if DR7.Global_Enable is set */
3003 return dr7 & (1 << 13);
3006 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3008 int dr = ctxt->modrm_reg;
3012 return emulate_ud(ctxt);
3014 cr4 = ctxt->ops->get_cr(ctxt, 4);
3015 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3016 return emulate_ud(ctxt);
3018 if (check_dr7_gd(ctxt))
3019 return emulate_db(ctxt);
3021 return X86EMUL_CONTINUE;
3024 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3026 u64 new_val = ctxt->src.val64;
3027 int dr = ctxt->modrm_reg;
3029 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3030 return emulate_gp(ctxt, 0);
3032 return check_dr_read(ctxt);
3035 static int check_svme(struct x86_emulate_ctxt *ctxt)
3039 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3041 if (!(efer & EFER_SVME))
3042 return emulate_ud(ctxt);
3044 return X86EMUL_CONTINUE;
3047 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3049 u64 rax = ctxt->regs[VCPU_REGS_RAX];
3051 /* Valid physical address? */
3052 if (rax & 0xffff000000000000ULL)
3053 return emulate_gp(ctxt, 0);
3055 return check_svme(ctxt);
3058 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3060 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3062 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3063 return emulate_ud(ctxt);
3065 return X86EMUL_CONTINUE;
3068 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3070 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3071 u64 rcx = ctxt->regs[VCPU_REGS_RCX];
3073 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3075 return emulate_gp(ctxt, 0);
3077 return X86EMUL_CONTINUE;
3080 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3082 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3083 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3084 return emulate_gp(ctxt, 0);
3086 return X86EMUL_CONTINUE;
3089 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3091 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3092 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3093 return emulate_gp(ctxt, 0);
3095 return X86EMUL_CONTINUE;
3098 #define D(_y) { .flags = (_y) }
3099 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3100 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3101 .check_perm = (_p) }
3103 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3104 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
3105 #define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
3106 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3107 #define II(_f, _e, _i) \
3108 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3109 #define IIP(_f, _e, _i, _p) \
3110 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3111 .check_perm = (_p) }
3112 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3114 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3115 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3116 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3118 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3119 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3120 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3122 static struct opcode group7_rm1[] = {
3123 DI(SrcNone | ModRM | Priv, monitor),
3124 DI(SrcNone | ModRM | Priv, mwait),
3128 static struct opcode group7_rm3[] = {
3129 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
3130 II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
3131 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
3132 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
3133 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
3134 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
3135 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
3136 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
3139 static struct opcode group7_rm7[] = {
3141 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
3145 static struct opcode group1[] = {
3156 static struct opcode group1A[] = {
3157 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
3160 static struct opcode group3[] = {
3161 I(DstMem | SrcImm | ModRM, em_test),
3162 I(DstMem | SrcImm | ModRM, em_test),
3163 I(DstMem | SrcNone | ModRM | Lock, em_not),
3164 I(DstMem | SrcNone | ModRM | Lock, em_neg),
3165 I(SrcMem | ModRM, em_mul_ex),
3166 I(SrcMem | ModRM, em_imul_ex),
3167 I(SrcMem | ModRM, em_div_ex),
3168 I(SrcMem | ModRM, em_idiv_ex),
3171 static struct opcode group4[] = {
3172 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
3176 static struct opcode group5[] = {
3177 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3178 D(SrcMem | ModRM | Stack),
3179 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3180 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
3181 D(SrcMem | ModRM | Stack), N,
3184 static struct opcode group6[] = {
3185 DI(ModRM | Prot, sldt),
3186 DI(ModRM | Prot, str),
3187 DI(ModRM | Prot | Priv, lldt),
3188 DI(ModRM | Prot | Priv, ltr),
3192 static struct group_dual group7 = { {
3193 DI(ModRM | Mov | DstMem | Priv, sgdt),
3194 DI(ModRM | Mov | DstMem | Priv, sidt),
3195 II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
3196 II(ModRM | SrcMem | Priv, em_lidt, lidt),
3197 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3198 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
3199 II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3201 I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
3203 N, EXT(0, group7_rm3),
3204 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3205 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3208 static struct opcode group8[] = {
3210 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
3211 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
3214 static struct group_dual group9 = { {
3215 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
3217 N, N, N, N, N, N, N, N,
3220 static struct opcode group11[] = {
3221 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
3224 static struct gprefix pfx_0f_6f_0f_7f = {
3225 N, N, N, I(Sse, em_movdqu),
3228 static struct opcode opcode_table[256] = {
3230 I6ALU(Lock, em_add),
3231 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3232 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3235 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3238 I6ALU(Lock, em_adc),
3239 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3240 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3242 I6ALU(Lock, em_sbb),
3243 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3244 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3246 I6ALU(Lock, em_and), N, N,
3248 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3250 I6ALU(Lock, em_xor), N, N,
3252 I6ALU(0, em_cmp), N, N,
3256 X8(I(SrcReg | Stack, em_push)),
3258 X8(I(DstReg | Stack, em_pop)),
3260 I(ImplicitOps | Stack | No64, em_pusha),
3261 I(ImplicitOps | Stack | No64, em_popa),
3262 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3265 I(SrcImm | Mov | Stack, em_push),
3266 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3267 I(SrcImmByte | Mov | Stack, em_push),
3268 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3269 D2bvIP(DstDI | SrcDX | Mov | String, ins, check_perm_in), /* insb, insw/insd */
3270 D2bvIP(SrcSI | DstDX | String, outs, check_perm_out), /* outsb, outsw/outsd */
3274 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
3275 G(DstMem | SrcImm | ModRM | Group, group1),
3276 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
3277 G(DstMem | SrcImmByte | ModRM | Group, group1),
3278 I2bv(DstMem | SrcReg | ModRM, em_test),
3279 I2bv(DstMem | SrcReg | ModRM | Lock, em_xchg),
3281 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
3282 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3283 I(DstMem | SrcNone | ModRM | Mov, em_mov_rm_sreg),
3284 D(ModRM | SrcMem | NoAccess | DstReg),
3285 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3288 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3290 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3291 I(SrcImmFAddr | No64, em_call_far), N,
3292 II(ImplicitOps | Stack, em_pushf, pushf),
3293 II(ImplicitOps | Stack, em_popf, popf), N, N,
3295 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3296 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
3297 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3298 I2bv(SrcSI | DstDI | String, em_cmp),
3300 I2bv(DstAcc | SrcImm, em_test),
3301 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3302 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3303 I2bv(SrcAcc | DstDI | String, em_cmp),
3305 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3307 X8(I(DstReg | SrcImm | Mov, em_mov)),
3309 D2bv(DstMem | SrcImmByte | ModRM),
3310 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3311 I(ImplicitOps | Stack, em_ret),
3312 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3313 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3314 G(ByteOp, group11), G(0, group11),
3316 N, N, N, I(ImplicitOps | Stack, em_ret_far),
3317 D(ImplicitOps), DI(SrcImmByte, intn),
3318 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3320 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3323 N, N, N, N, N, N, N, N,
3325 X3(I(SrcImmByte, em_loop)),
3326 I(SrcImmByte, em_jcxz),
3327 D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
3328 D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
3330 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3331 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3332 D2bvIP(SrcDX | DstAcc, in, check_perm_in),
3333 D2bvIP(SrcAcc | DstDX, out, check_perm_out),
3335 N, DI(ImplicitOps, icebp), N, N,
3336 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3337 G(ByteOp, group3), G(0, group3),
3339 D(ImplicitOps), D(ImplicitOps),
3340 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3341 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3344 static struct opcode twobyte_table[256] = {
3346 G(0, group6), GD(0, &group7), N, N,
3347 N, I(ImplicitOps | VendorSpecific, em_syscall),
3348 II(ImplicitOps | Priv, em_clts, clts), N,
3349 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3350 N, D(ImplicitOps | ModRM), N, N,
3352 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3354 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3355 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3356 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3357 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3359 N, N, N, N, N, N, N, N,
3361 DI(ImplicitOps | Priv, wrmsr),
3362 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3363 DI(ImplicitOps | Priv, rdmsr),
3364 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3365 I(ImplicitOps | VendorSpecific, em_sysenter),
3366 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3368 N, N, N, N, N, N, N, N,
3370 X16(D(DstReg | SrcMem | ModRM | Mov)),
3372 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3377 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3382 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3386 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3388 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3389 DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3390 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3391 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3393 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3394 DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3395 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3396 D(DstMem | SrcReg | Src2CL | ModRM),
3397 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3399 D2bv(DstMem | SrcReg | ModRM | Lock),
3400 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3401 D(DstMem | SrcReg | ModRM | BitOp | Lock),
3402 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3403 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3404 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3407 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3408 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3409 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3411 D2bv(DstMem | SrcReg | ModRM | Lock),
3412 N, D(DstMem | SrcReg | ModRM | Mov),
3413 N, N, N, GD(0, &group9),
3414 N, N, N, N, N, N, N, N,
3416 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3418 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3420 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3436 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3440 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3446 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3447 unsigned size, bool sign_extension)
3449 int rc = X86EMUL_CONTINUE;
3453 op->addr.mem.ea = ctxt->_eip;
3454 /* NB. Immediates are sign-extended as necessary. */
3455 switch (op->bytes) {
3457 op->val = insn_fetch(s8, ctxt);
3460 op->val = insn_fetch(s16, ctxt);
3463 op->val = insn_fetch(s32, ctxt);
3466 if (!sign_extension) {
3467 switch (op->bytes) {
3475 op->val &= 0xffffffff;
3483 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
3486 int rc = X86EMUL_CONTINUE;
3490 decode_register_operand(ctxt, op,
3492 ctxt->twobyte && (ctxt->b == 0xb6 || ctxt->b == 0xb7));
3495 rc = decode_imm(ctxt, op, 1, false);
3498 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3502 if ((ctxt->d & BitOp) && op == &ctxt->dst)
3503 fetch_bit_operand(ctxt);
3504 op->orig_val = op->val;
3507 ctxt->memop.bytes = 8;
3511 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3512 op->addr.reg = &ctxt->regs[VCPU_REGS_RAX];
3513 fetch_register_operand(op);
3514 op->orig_val = op->val;
3518 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3520 register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]);
3521 op->addr.mem.seg = VCPU_SREG_ES;
3527 op->addr.reg = &ctxt->regs[VCPU_REGS_RDX];
3528 fetch_register_operand(op);
3532 op->val = ctxt->regs[VCPU_REGS_RCX] & 0xff;
3535 rc = decode_imm(ctxt, op, 1, true);
3542 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
3545 ctxt->memop.bytes = 2;
3548 ctxt->memop.bytes = 4;
3551 rc = decode_imm(ctxt, op, 2, false);
3554 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
3558 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3560 register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]);
3561 op->addr.mem.seg = seg_override(ctxt);
3566 op->addr.mem.ea = ctxt->_eip;
3567 op->bytes = ctxt->op_bytes + 2;
3568 insn_fetch_arr(op->valptr, op->bytes, ctxt);
3571 ctxt->memop.bytes = ctxt->op_bytes + 2;
3574 op->val = VCPU_SREG_ES;
3577 op->val = VCPU_SREG_CS;
3580 op->val = VCPU_SREG_SS;
3583 op->val = VCPU_SREG_DS;
3586 op->val = VCPU_SREG_FS;
3589 op->val = VCPU_SREG_GS;
3592 /* Special instructions do their own operand decoding. */
3594 op->type = OP_NONE; /* Disable writeback. */
3602 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3604 int rc = X86EMUL_CONTINUE;
3605 int mode = ctxt->mode;
3606 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3607 bool op_prefix = false;
3608 struct opcode opcode;
3610 ctxt->memop.type = OP_NONE;
3611 ctxt->memopp = NULL;
3612 ctxt->_eip = ctxt->eip;
3613 ctxt->fetch.start = ctxt->_eip;
3614 ctxt->fetch.end = ctxt->fetch.start + insn_len;
3616 memcpy(ctxt->fetch.data, insn, insn_len);
3619 case X86EMUL_MODE_REAL:
3620 case X86EMUL_MODE_VM86:
3621 case X86EMUL_MODE_PROT16:
3622 def_op_bytes = def_ad_bytes = 2;
3624 case X86EMUL_MODE_PROT32:
3625 def_op_bytes = def_ad_bytes = 4;
3627 #ifdef CONFIG_X86_64
3628 case X86EMUL_MODE_PROT64:
3634 return EMULATION_FAILED;
3637 ctxt->op_bytes = def_op_bytes;
3638 ctxt->ad_bytes = def_ad_bytes;
3640 /* Legacy prefixes. */
3642 switch (ctxt->b = insn_fetch(u8, ctxt)) {
3643 case 0x66: /* operand-size override */
3645 /* switch between 2/4 bytes */
3646 ctxt->op_bytes = def_op_bytes ^ 6;
3648 case 0x67: /* address-size override */
3649 if (mode == X86EMUL_MODE_PROT64)
3650 /* switch between 4/8 bytes */
3651 ctxt->ad_bytes = def_ad_bytes ^ 12;
3653 /* switch between 2/4 bytes */
3654 ctxt->ad_bytes = def_ad_bytes ^ 6;
3656 case 0x26: /* ES override */
3657 case 0x2e: /* CS override */
3658 case 0x36: /* SS override */
3659 case 0x3e: /* DS override */
3660 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
3662 case 0x64: /* FS override */
3663 case 0x65: /* GS override */
3664 set_seg_override(ctxt, ctxt->b & 7);
3666 case 0x40 ... 0x4f: /* REX */
3667 if (mode != X86EMUL_MODE_PROT64)
3669 ctxt->rex_prefix = ctxt->b;
3671 case 0xf0: /* LOCK */
3672 ctxt->lock_prefix = 1;
3674 case 0xf2: /* REPNE/REPNZ */
3675 case 0xf3: /* REP/REPE/REPZ */
3676 ctxt->rep_prefix = ctxt->b;
3682 /* Any legacy prefix after a REX prefix nullifies its effect. */
3684 ctxt->rex_prefix = 0;
3690 if (ctxt->rex_prefix & 8)
3691 ctxt->op_bytes = 8; /* REX.W */
3693 /* Opcode byte(s). */
3694 opcode = opcode_table[ctxt->b];
3695 /* Two-byte opcode? */
3696 if (ctxt->b == 0x0f) {
3698 ctxt->b = insn_fetch(u8, ctxt);
3699 opcode = twobyte_table[ctxt->b];
3701 ctxt->d = opcode.flags;
3703 while (ctxt->d & GroupMask) {
3704 switch (ctxt->d & GroupMask) {
3706 ctxt->modrm = insn_fetch(u8, ctxt);
3708 goffset = (ctxt->modrm >> 3) & 7;
3709 opcode = opcode.u.group[goffset];
3712 ctxt->modrm = insn_fetch(u8, ctxt);
3714 goffset = (ctxt->modrm >> 3) & 7;
3715 if ((ctxt->modrm >> 6) == 3)
3716 opcode = opcode.u.gdual->mod3[goffset];
3718 opcode = opcode.u.gdual->mod012[goffset];
3721 goffset = ctxt->modrm & 7;
3722 opcode = opcode.u.group[goffset];
3725 if (ctxt->rep_prefix && op_prefix)
3726 return EMULATION_FAILED;
3727 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
3728 switch (simd_prefix) {
3729 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3730 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3731 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3732 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3736 return EMULATION_FAILED;
3739 ctxt->d &= ~(u64)GroupMask;
3740 ctxt->d |= opcode.flags;
3743 ctxt->execute = opcode.u.execute;
3744 ctxt->check_perm = opcode.check_perm;
3745 ctxt->intercept = opcode.intercept;
3748 if (ctxt->d == 0 || (ctxt->d & Undefined))
3749 return EMULATION_FAILED;
3751 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3752 return EMULATION_FAILED;
3754 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
3757 if (ctxt->d & Op3264) {
3758 if (mode == X86EMUL_MODE_PROT64)
3765 ctxt->op_bytes = 16;
3767 /* ModRM and SIB bytes. */
3768 if (ctxt->d & ModRM) {
3769 rc = decode_modrm(ctxt, &ctxt->memop);
3770 if (!ctxt->has_seg_override)
3771 set_seg_override(ctxt, ctxt->modrm_seg);
3772 } else if (ctxt->d & MemAbs)
3773 rc = decode_abs(ctxt, &ctxt->memop);
3774 if (rc != X86EMUL_CONTINUE)
3777 if (!ctxt->has_seg_override)
3778 set_seg_override(ctxt, VCPU_SREG_DS);
3780 ctxt->memop.addr.mem.seg = seg_override(ctxt);
3782 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
3783 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
3786 * Decode and fetch the source operand: register, memory
3789 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
3790 if (rc != X86EMUL_CONTINUE)
3794 * Decode and fetch the second source operand: register, memory
3797 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
3798 if (rc != X86EMUL_CONTINUE)
3801 /* Decode and fetch the destination operand: register or memory. */
3802 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
3805 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
3806 ctxt->memopp->addr.mem.ea += ctxt->_eip;
3808 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
3811 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3813 /* The second termination condition only applies for REPE
3814 * and REPNE. Test if the repeat string operation prefix is
3815 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3816 * corresponding termination condition according to:
3817 * - if REPE/REPZ and ZF = 0 then done
3818 * - if REPNE/REPNZ and ZF = 1 then done
3820 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
3821 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
3822 && (((ctxt->rep_prefix == REPE_PREFIX) &&
3823 ((ctxt->eflags & EFLG_ZF) == 0))
3824 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
3825 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3831 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3833 struct x86_emulate_ops *ops = ctxt->ops;
3835 int rc = X86EMUL_CONTINUE;
3836 int saved_dst_type = ctxt->dst.type;
3838 ctxt->mem_read.pos = 0;
3840 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
3841 rc = emulate_ud(ctxt);
3845 /* LOCK prefix is allowed only with some instructions */
3846 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
3847 rc = emulate_ud(ctxt);
3851 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
3852 rc = emulate_ud(ctxt);
3857 && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
3858 || !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
3859 rc = emulate_ud(ctxt);
3863 if ((ctxt->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
3864 rc = emulate_nm(ctxt);
3868 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3869 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3870 X86_ICPT_PRE_EXCEPT);
3871 if (rc != X86EMUL_CONTINUE)
3875 /* Privileged instruction can be executed only in CPL=0 */
3876 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
3877 rc = emulate_gp(ctxt, 0);
3881 /* Instruction can only be executed in protected mode */
3882 if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3883 rc = emulate_ud(ctxt);
3887 /* Do instruction specific permission checks */
3888 if (ctxt->check_perm) {
3889 rc = ctxt->check_perm(ctxt);
3890 if (rc != X86EMUL_CONTINUE)
3894 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3895 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3896 X86_ICPT_POST_EXCEPT);
3897 if (rc != X86EMUL_CONTINUE)
3901 if (ctxt->rep_prefix && (ctxt->d & String)) {
3902 /* All REP prefixes have the same first termination condition */
3903 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) {
3904 ctxt->eip = ctxt->_eip;
3909 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
3910 rc = segmented_read(ctxt, ctxt->src.addr.mem,
3911 ctxt->src.valptr, ctxt->src.bytes);
3912 if (rc != X86EMUL_CONTINUE)
3914 ctxt->src.orig_val64 = ctxt->src.val64;
3917 if (ctxt->src2.type == OP_MEM) {
3918 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
3919 &ctxt->src2.val, ctxt->src2.bytes);
3920 if (rc != X86EMUL_CONTINUE)
3924 if ((ctxt->d & DstMask) == ImplicitOps)
3928 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
3929 /* optimisation - avoid slow emulated read if Mov */
3930 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
3931 &ctxt->dst.val, ctxt->dst.bytes);
3932 if (rc != X86EMUL_CONTINUE)
3935 ctxt->dst.orig_val = ctxt->dst.val;
3939 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3940 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3941 X86_ICPT_POST_MEMACCESS);
3942 if (rc != X86EMUL_CONTINUE)
3946 if (ctxt->execute) {
3947 rc = ctxt->execute(ctxt);
3948 if (rc != X86EMUL_CONTINUE)
3957 case 0x40 ... 0x47: /* inc r16/r32 */
3958 emulate_1op(ctxt, "inc");
3960 case 0x48 ... 0x4f: /* dec r16/r32 */
3961 emulate_1op(ctxt, "dec");
3963 case 0x63: /* movsxd */
3964 if (ctxt->mode != X86EMUL_MODE_PROT64)
3965 goto cannot_emulate;
3966 ctxt->dst.val = (s32) ctxt->src.val;
3968 case 0x6c: /* insb */
3969 case 0x6d: /* insw/insd */
3970 ctxt->src.val = ctxt->regs[VCPU_REGS_RDX];
3972 case 0x6e: /* outsb */
3973 case 0x6f: /* outsw/outsd */
3974 ctxt->dst.val = ctxt->regs[VCPU_REGS_RDX];
3977 case 0x70 ... 0x7f: /* jcc (short) */
3978 if (test_cc(ctxt->b, ctxt->eflags))
3979 rc = jmp_rel(ctxt, ctxt->src.val);
3981 case 0x8d: /* lea r16/r32, m */
3982 ctxt->dst.val = ctxt->src.addr.mem.ea;
3984 case 0x8f: /* pop (sole member of Grp1a) */
3985 rc = em_grp1a(ctxt);
3987 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3988 if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX])
3992 case 0x98: /* cbw/cwde/cdqe */
3993 switch (ctxt->op_bytes) {
3994 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
3995 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
3996 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4002 case 0xcc: /* int3 */
4003 rc = emulate_int(ctxt, 3);
4005 case 0xcd: /* int n */
4006 rc = emulate_int(ctxt, ctxt->src.val);
4008 case 0xce: /* into */
4009 if (ctxt->eflags & EFLG_OF)
4010 rc = emulate_int(ctxt, 4);
4012 case 0xd0 ... 0xd1: /* Grp2 */
4015 case 0xd2 ... 0xd3: /* Grp2 */
4016 ctxt->src.val = ctxt->regs[VCPU_REGS_RCX];
4019 case 0xe4: /* inb */
4022 case 0xe6: /* outb */
4023 case 0xe7: /* out */
4025 case 0xe9: /* jmp rel */
4026 case 0xeb: /* jmp rel short */
4027 rc = jmp_rel(ctxt, ctxt->src.val);
4028 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4030 case 0xec: /* in al,dx */
4031 case 0xed: /* in (e/r)ax,dx */
4033 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
4035 goto done; /* IO is needed */
4037 case 0xee: /* out dx,al */
4038 case 0xef: /* out dx,(e/r)ax */
4040 ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
4042 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4044 case 0xf4: /* hlt */
4045 ctxt->ops->halt(ctxt);
4047 case 0xf5: /* cmc */
4048 /* complement carry flag from eflags reg */
4049 ctxt->eflags ^= EFLG_CF;
4051 case 0xf8: /* clc */
4052 ctxt->eflags &= ~EFLG_CF;
4054 case 0xf9: /* stc */
4055 ctxt->eflags |= EFLG_CF;
4057 case 0xfc: /* cld */
4058 ctxt->eflags &= ~EFLG_DF;
4060 case 0xfd: /* std */
4061 ctxt->eflags |= EFLG_DF;
4063 case 0xfe: /* Grp4 */
4064 rc = em_grp45(ctxt);
4066 case 0xff: /* Grp5 */
4067 rc = em_grp45(ctxt);
4070 goto cannot_emulate;
4073 if (rc != X86EMUL_CONTINUE)
4077 rc = writeback(ctxt);
4078 if (rc != X86EMUL_CONTINUE)
4082 * restore dst type in case the decoding will be reused
4083 * (happens for string instruction )
4085 ctxt->dst.type = saved_dst_type;
4087 if ((ctxt->d & SrcMask) == SrcSI)
4088 string_addr_inc(ctxt, seg_override(ctxt),
4089 VCPU_REGS_RSI, &ctxt->src);
4091 if ((ctxt->d & DstMask) == DstDI)
4092 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4095 if (ctxt->rep_prefix && (ctxt->d & String)) {
4096 struct read_cache *r = &ctxt->io_read;
4097 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
4099 if (!string_insn_completed(ctxt)) {
4101 * Re-enter guest when pio read ahead buffer is empty
4102 * or, if it is not used, after each 1024 iteration.
4104 if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) &&
4105 (r->end == 0 || r->end != r->pos)) {
4107 * Reset read cache. Usually happens before
4108 * decode, but since instruction is restarted
4109 * we have to do it here.
4111 ctxt->mem_read.end = 0;
4112 return EMULATION_RESTART;
4114 goto done; /* skip rip writeback */
4118 ctxt->eip = ctxt->_eip;
4121 if (rc == X86EMUL_PROPAGATE_FAULT)
4122 ctxt->have_exception = true;
4123 if (rc == X86EMUL_INTERCEPTED)
4124 return EMULATION_INTERCEPTED;
4126 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4130 case 0x09: /* wbinvd */
4131 (ctxt->ops->wbinvd)(ctxt);
4133 case 0x08: /* invd */
4134 case 0x0d: /* GrpP (prefetch) */
4135 case 0x18: /* Grp16 (prefetch/nop) */
4137 case 0x20: /* mov cr, reg */
4138 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4140 case 0x21: /* mov from dr to reg */
4141 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4143 case 0x22: /* mov reg, cr */
4144 if (ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val)) {
4145 emulate_gp(ctxt, 0);
4146 rc = X86EMUL_PROPAGATE_FAULT;
4149 ctxt->dst.type = OP_NONE;
4151 case 0x23: /* mov from reg to dr */
4152 if (ops->set_dr(ctxt, ctxt->modrm_reg, ctxt->src.val &
4153 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4154 ~0ULL : ~0U)) < 0) {
4155 /* #UD condition is already handled by the code above */
4156 emulate_gp(ctxt, 0);
4157 rc = X86EMUL_PROPAGATE_FAULT;
4161 ctxt->dst.type = OP_NONE; /* no writeback */
4165 msr_data = (u32)ctxt->regs[VCPU_REGS_RAX]
4166 | ((u64)ctxt->regs[VCPU_REGS_RDX] << 32);
4167 if (ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data)) {
4168 emulate_gp(ctxt, 0);
4169 rc = X86EMUL_PROPAGATE_FAULT;
4172 rc = X86EMUL_CONTINUE;
4176 if (ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data)) {
4177 emulate_gp(ctxt, 0);
4178 rc = X86EMUL_PROPAGATE_FAULT;
4181 ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data;
4182 ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32;
4184 rc = X86EMUL_CONTINUE;
4186 case 0x40 ... 0x4f: /* cmov */
4187 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4188 if (!test_cc(ctxt->b, ctxt->eflags))
4189 ctxt->dst.type = OP_NONE; /* no writeback */
4191 case 0x80 ... 0x8f: /* jnz rel, etc*/
4192 if (test_cc(ctxt->b, ctxt->eflags))
4193 rc = jmp_rel(ctxt, ctxt->src.val);
4195 case 0x90 ... 0x9f: /* setcc r/m8 */
4196 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4200 ctxt->dst.type = OP_NONE;
4201 /* only subword offset */
4202 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
4203 emulate_2op_SrcV_nobyte(ctxt, "bt");
4205 case 0xa4: /* shld imm8, r, r/m */
4206 case 0xa5: /* shld cl, r, r/m */
4207 emulate_2op_cl(ctxt, "shld");
4211 emulate_2op_SrcV_nobyte(ctxt, "bts");
4213 case 0xac: /* shrd imm8, r, r/m */
4214 case 0xad: /* shrd cl, r, r/m */
4215 emulate_2op_cl(ctxt, "shrd");
4217 case 0xae: /* clflush */
4219 case 0xb0 ... 0xb1: /* cmpxchg */
4221 * Save real source value, then compare EAX against
4224 ctxt->src.orig_val = ctxt->src.val;
4225 ctxt->src.val = ctxt->regs[VCPU_REGS_RAX];
4226 emulate_2op_SrcV(ctxt, "cmp");
4227 if (ctxt->eflags & EFLG_ZF) {
4228 /* Success: write back to memory. */
4229 ctxt->dst.val = ctxt->src.orig_val;
4231 /* Failure: write the value we saw to EAX. */
4232 ctxt->dst.type = OP_REG;
4233 ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX];
4238 emulate_2op_SrcV_nobyte(ctxt, "btr");
4240 case 0xb6 ... 0xb7: /* movzx */
4241 ctxt->dst.bytes = ctxt->op_bytes;
4242 ctxt->dst.val = (ctxt->d & ByteOp) ? (u8) ctxt->src.val
4243 : (u16) ctxt->src.val;
4245 case 0xba: /* Grp8 */
4246 switch (ctxt->modrm_reg & 3) {
4259 emulate_2op_SrcV_nobyte(ctxt, "btc");
4261 case 0xbc: { /* bsf */
4263 __asm__ ("bsf %2, %0; setz %1"
4264 : "=r"(ctxt->dst.val), "=q"(zf)
4265 : "r"(ctxt->src.val));
4266 ctxt->eflags &= ~X86_EFLAGS_ZF;
4268 ctxt->eflags |= X86_EFLAGS_ZF;
4269 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4273 case 0xbd: { /* bsr */
4275 __asm__ ("bsr %2, %0; setz %1"
4276 : "=r"(ctxt->dst.val), "=q"(zf)
4277 : "r"(ctxt->src.val));
4278 ctxt->eflags &= ~X86_EFLAGS_ZF;
4280 ctxt->eflags |= X86_EFLAGS_ZF;
4281 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4285 case 0xbe ... 0xbf: /* movsx */
4286 ctxt->dst.bytes = ctxt->op_bytes;
4287 ctxt->dst.val = (ctxt->d & ByteOp) ? (s8) ctxt->src.val :
4288 (s16) ctxt->src.val;
4290 case 0xc0 ... 0xc1: /* xadd */
4291 emulate_2op_SrcV(ctxt, "add");
4292 /* Write back the register source. */
4293 ctxt->src.val = ctxt->dst.orig_val;
4294 write_register_operand(&ctxt->src);
4296 case 0xc3: /* movnti */
4297 ctxt->dst.bytes = ctxt->op_bytes;
4298 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4299 (u64) ctxt->src.val;
4301 case 0xc7: /* Grp9 (cmpxchg8b) */
4305 goto cannot_emulate;
4308 if (rc != X86EMUL_CONTINUE)
4314 return EMULATION_FAILED;