1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
35 #define OpImplicit 1ull /* No generic decode */
36 #define OpReg 2ull /* Register */
37 #define OpMem 3ull /* Memory */
38 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
39 #define OpDI 5ull /* ES:DI/EDI/RDI */
40 #define OpMem64 6ull /* Memory, 64-bit */
41 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
42 #define OpDX 8ull /* DX register */
43 #define OpCL 9ull /* CL register (for shifts) */
44 #define OpImmByte 10ull /* 8-bit sign extended immediate */
45 #define OpOne 11ull /* Implied 1 */
46 #define OpImm 12ull /* Sign extended immediate */
47 #define OpMem16 13ull /* Memory operand (16-bit). */
48 #define OpMem32 14ull /* Memory operand (32-bit). */
49 #define OpImmU 15ull /* Immediate operand, zero extended */
50 #define OpSI 16ull /* SI/ESI/RSI */
51 #define OpImmFAddr 17ull /* Immediate far address */
52 #define OpMemFAddr 18ull /* Far address in memory */
53 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
54 #define OpES 20ull /* ES */
55 #define OpCS 21ull /* CS */
56 #define OpSS 22ull /* SS */
57 #define OpDS 23ull /* DS */
58 #define OpFS 24ull /* FS */
59 #define OpGS 25ull /* GS */
61 #define OpBits 5 /* Width of operand field */
62 #define OpMask ((1ull << OpBits) - 1)
65 * Opcode effective-address decode tables.
66 * Note that we only emulate instructions that have at least one memory
67 * operand (excluding implicit stack references). We assume that stack
68 * references and instruction fetches will never occur in special memory
69 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
73 /* Operand sizes: 8-bit operands or specified/overridden size. */
74 #define ByteOp (1<<0) /* 8-bit operands. */
75 /* Destination operand type. */
77 #define ImplicitOps (OpImplicit << DstShift)
78 #define DstReg (OpReg << DstShift)
79 #define DstMem (OpMem << DstShift)
80 #define DstAcc (OpAcc << DstShift)
81 #define DstDI (OpDI << DstShift)
82 #define DstMem64 (OpMem64 << DstShift)
83 #define DstImmUByte (OpImmUByte << DstShift)
84 #define DstDX (OpDX << DstShift)
85 #define DstMask (OpMask << DstShift)
86 /* Source operand type. */
88 #define SrcNone (OpNone << SrcShift)
89 #define SrcReg (OpReg << SrcShift)
90 #define SrcMem (OpMem << SrcShift)
91 #define SrcMem16 (OpMem16 << SrcShift)
92 #define SrcMem32 (OpMem32 << SrcShift)
93 #define SrcImm (OpImm << SrcShift)
94 #define SrcImmByte (OpImmByte << SrcShift)
95 #define SrcOne (OpOne << SrcShift)
96 #define SrcImmUByte (OpImmUByte << SrcShift)
97 #define SrcImmU (OpImmU << SrcShift)
98 #define SrcSI (OpSI << SrcShift)
99 #define SrcImmFAddr (OpImmFAddr << SrcShift)
100 #define SrcMemFAddr (OpMemFAddr << SrcShift)
101 #define SrcAcc (OpAcc << SrcShift)
102 #define SrcImmU16 (OpImmU16 << SrcShift)
103 #define SrcDX (OpDX << SrcShift)
104 #define SrcMask (OpMask << SrcShift)
105 #define BitOp (1<<11)
106 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
107 #define String (1<<13) /* String instruction (rep capable) */
108 #define Stack (1<<14) /* Stack instruction (push/pop) */
109 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
110 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
111 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
112 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
113 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
114 #define Sse (1<<18) /* SSE Vector instruction */
115 /* Generic ModRM decode. */
116 #define ModRM (1<<19)
117 /* Destination is only written; never read. */
120 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
121 #define VendorSpecific (1<<22) /* Vendor specific instruction */
122 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
123 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
124 #define Undefined (1<<25) /* No Such Instruction */
125 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
126 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
128 /* Source 2 operand type */
129 #define Src2Shift (29)
130 #define Src2None (OpNone << Src2Shift)
131 #define Src2CL (OpCL << Src2Shift)
132 #define Src2ImmByte (OpImmByte << Src2Shift)
133 #define Src2One (OpOne << Src2Shift)
134 #define Src2Imm (OpImm << Src2Shift)
135 #define Src2ES (OpES << Src2Shift)
136 #define Src2CS (OpCS << Src2Shift)
137 #define Src2SS (OpSS << Src2Shift)
138 #define Src2DS (OpDS << Src2Shift)
139 #define Src2FS (OpFS << Src2Shift)
140 #define Src2GS (OpGS << Src2Shift)
141 #define Src2Mask (OpMask << Src2Shift)
143 #define X2(x...) x, x
144 #define X3(x...) X2(x), x
145 #define X4(x...) X2(x), X2(x)
146 #define X5(x...) X4(x), x
147 #define X6(x...) X4(x), X2(x)
148 #define X7(x...) X4(x), X3(x)
149 #define X8(x...) X4(x), X4(x)
150 #define X16(x...) X8(x), X8(x)
156 int (*execute)(struct x86_emulate_ctxt *ctxt);
157 struct opcode *group;
158 struct group_dual *gdual;
159 struct gprefix *gprefix;
161 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
165 struct opcode mod012[8];
166 struct opcode mod3[8];
170 struct opcode pfx_no;
171 struct opcode pfx_66;
172 struct opcode pfx_f2;
173 struct opcode pfx_f3;
176 /* EFLAGS bit definitions. */
177 #define EFLG_ID (1<<21)
178 #define EFLG_VIP (1<<20)
179 #define EFLG_VIF (1<<19)
180 #define EFLG_AC (1<<18)
181 #define EFLG_VM (1<<17)
182 #define EFLG_RF (1<<16)
183 #define EFLG_IOPL (3<<12)
184 #define EFLG_NT (1<<14)
185 #define EFLG_OF (1<<11)
186 #define EFLG_DF (1<<10)
187 #define EFLG_IF (1<<9)
188 #define EFLG_TF (1<<8)
189 #define EFLG_SF (1<<7)
190 #define EFLG_ZF (1<<6)
191 #define EFLG_AF (1<<4)
192 #define EFLG_PF (1<<2)
193 #define EFLG_CF (1<<0)
195 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
196 #define EFLG_RESERVED_ONE_MASK 2
199 * Instruction emulation:
200 * Most instructions are emulated directly via a fragment of inline assembly
201 * code. This allows us to save/restore EFLAGS and thus very easily pick up
202 * any modified flags.
205 #if defined(CONFIG_X86_64)
206 #define _LO32 "k" /* force 32-bit operand */
207 #define _STK "%%rsp" /* stack pointer */
208 #elif defined(__i386__)
209 #define _LO32 "" /* force 32-bit operand */
210 #define _STK "%%esp" /* stack pointer */
214 * These EFLAGS bits are restored from saved value during emulation, and
215 * any changes are written back to the saved value after emulation.
217 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
219 /* Before executing instruction: restore necessary bits in EFLAGS. */
220 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
221 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
222 "movl %"_sav",%"_LO32 _tmp"; " \
225 "movl %"_msk",%"_LO32 _tmp"; " \
226 "andl %"_LO32 _tmp",("_STK"); " \
228 "notl %"_LO32 _tmp"; " \
229 "andl %"_LO32 _tmp",("_STK"); " \
230 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
232 "orl %"_LO32 _tmp",("_STK"); " \
236 /* After executing instruction: write-back necessary bits in EFLAGS. */
237 #define _POST_EFLAGS(_sav, _msk, _tmp) \
238 /* _sav |= EFLAGS & _msk; */ \
241 "andl %"_msk",%"_LO32 _tmp"; " \
242 "orl %"_LO32 _tmp",%"_sav"; "
250 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
252 __asm__ __volatile__ ( \
253 _PRE_EFLAGS("0", "4", "2") \
254 _op _suffix " %"_x"3,%1; " \
255 _POST_EFLAGS("0", "4", "2") \
256 : "=m" ((ctxt)->eflags), \
257 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
259 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
263 /* Raw emulation: instruction has two explicit operands. */
264 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
266 unsigned long _tmp; \
268 switch ((ctxt)->dst.bytes) { \
270 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
273 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
276 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
281 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
283 unsigned long _tmp; \
284 switch ((ctxt)->dst.bytes) { \
286 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
289 __emulate_2op_nobyte(ctxt, _op, \
290 _wx, _wy, _lx, _ly, _qx, _qy); \
295 /* Source operand is byte-sized and may be restricted to just %cl. */
296 #define emulate_2op_SrcB(ctxt, _op) \
297 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
299 /* Source operand is byte, word, long or quad sized. */
300 #define emulate_2op_SrcV(ctxt, _op) \
301 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
303 /* Source operand is word, long or quad sized. */
304 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
305 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
307 /* Instruction has three operands and one operand is stored in ECX register */
308 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
310 unsigned long _tmp; \
311 _type _clv = (ctxt)->src2.val; \
312 _type _srcv = (ctxt)->src.val; \
313 _type _dstv = (ctxt)->dst.val; \
315 __asm__ __volatile__ ( \
316 _PRE_EFLAGS("0", "5", "2") \
317 _op _suffix " %4,%1 \n" \
318 _POST_EFLAGS("0", "5", "2") \
319 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
320 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
323 (ctxt)->src2.val = (unsigned long) _clv; \
324 (ctxt)->src2.val = (unsigned long) _srcv; \
325 (ctxt)->dst.val = (unsigned long) _dstv; \
328 #define emulate_2op_cl(ctxt, _op) \
330 switch ((ctxt)->dst.bytes) { \
332 __emulate_2op_cl(ctxt, _op, "w", u16); \
335 __emulate_2op_cl(ctxt, _op, "l", u32); \
338 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
343 #define __emulate_1op(ctxt, _op, _suffix) \
345 unsigned long _tmp; \
347 __asm__ __volatile__ ( \
348 _PRE_EFLAGS("0", "3", "2") \
349 _op _suffix " %1; " \
350 _POST_EFLAGS("0", "3", "2") \
351 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
353 : "i" (EFLAGS_MASK)); \
356 /* Instruction has only one explicit operand (no source operand). */
357 #define emulate_1op(ctxt, _op) \
359 switch ((ctxt)->dst.bytes) { \
360 case 1: __emulate_1op(ctxt, _op, "b"); break; \
361 case 2: __emulate_1op(ctxt, _op, "w"); break; \
362 case 4: __emulate_1op(ctxt, _op, "l"); break; \
363 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
367 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
369 unsigned long _tmp; \
370 ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX]; \
371 ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX]; \
373 __asm__ __volatile__ ( \
374 _PRE_EFLAGS("0", "5", "1") \
376 _op _suffix " %6; " \
378 _POST_EFLAGS("0", "5", "1") \
379 ".pushsection .fixup,\"ax\" \n\t" \
380 "3: movb $1, %4 \n\t" \
383 _ASM_EXTABLE(1b, 3b) \
384 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
385 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
386 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val), \
387 "a" (*rax), "d" (*rdx)); \
390 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
391 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
393 switch((ctxt)->src.bytes) { \
395 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
398 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
401 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
404 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
409 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
410 enum x86_intercept intercept,
411 enum x86_intercept_stage stage)
413 struct x86_instruction_info info = {
414 .intercept = intercept,
415 .rep_prefix = ctxt->rep_prefix,
416 .modrm_mod = ctxt->modrm_mod,
417 .modrm_reg = ctxt->modrm_reg,
418 .modrm_rm = ctxt->modrm_rm,
419 .src_val = ctxt->src.val64,
420 .src_bytes = ctxt->src.bytes,
421 .dst_bytes = ctxt->dst.bytes,
422 .ad_bytes = ctxt->ad_bytes,
423 .next_rip = ctxt->eip,
426 return ctxt->ops->intercept(ctxt, &info, stage);
429 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
431 return (1UL << (ctxt->ad_bytes << 3)) - 1;
434 /* Access/update address held in a register, based on addressing mode. */
435 static inline unsigned long
436 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
438 if (ctxt->ad_bytes == sizeof(unsigned long))
441 return reg & ad_mask(ctxt);
444 static inline unsigned long
445 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
447 return address_mask(ctxt, reg);
451 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
453 if (ctxt->ad_bytes == sizeof(unsigned long))
456 *reg = (*reg & ~ad_mask(ctxt)) | ((*reg + inc) & ad_mask(ctxt));
459 static u32 desc_limit_scaled(struct desc_struct *desc)
461 u32 limit = get_desc_limit(desc);
463 return desc->g ? (limit << 12) | 0xfff : limit;
466 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
468 ctxt->has_seg_override = true;
469 ctxt->seg_override = seg;
472 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
474 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
477 return ctxt->ops->get_cached_segment_base(ctxt, seg);
480 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
482 if (!ctxt->has_seg_override)
485 return ctxt->seg_override;
488 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
489 u32 error, bool valid)
491 ctxt->exception.vector = vec;
492 ctxt->exception.error_code = error;
493 ctxt->exception.error_code_valid = valid;
494 return X86EMUL_PROPAGATE_FAULT;
497 static int emulate_db(struct x86_emulate_ctxt *ctxt)
499 return emulate_exception(ctxt, DB_VECTOR, 0, false);
502 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
504 return emulate_exception(ctxt, GP_VECTOR, err, true);
507 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
509 return emulate_exception(ctxt, SS_VECTOR, err, true);
512 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
514 return emulate_exception(ctxt, UD_VECTOR, 0, false);
517 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
519 return emulate_exception(ctxt, TS_VECTOR, err, true);
522 static int emulate_de(struct x86_emulate_ctxt *ctxt)
524 return emulate_exception(ctxt, DE_VECTOR, 0, false);
527 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
529 return emulate_exception(ctxt, NM_VECTOR, 0, false);
532 static inline void assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
534 switch (ctxt->op_bytes) {
536 ctxt->_eip = (u16)dst;
539 ctxt->_eip = (u32)dst;
545 WARN(1, "unsupported eip assignment size\n");
549 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
551 assign_eip_near(ctxt, ctxt->_eip + rel);
554 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
557 struct desc_struct desc;
559 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
563 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
568 struct desc_struct desc;
570 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
571 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
574 static int __linearize(struct x86_emulate_ctxt *ctxt,
575 struct segmented_address addr,
576 unsigned size, bool write, bool fetch,
579 struct desc_struct desc;
586 la = seg_base(ctxt, addr.seg) + addr.ea;
587 switch (ctxt->mode) {
588 case X86EMUL_MODE_REAL:
590 case X86EMUL_MODE_PROT64:
591 if (((signed long)la << 16) >> 16 != la)
592 return emulate_gp(ctxt, 0);
595 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
599 /* code segment or read-only data segment */
600 if (((desc.type & 8) || !(desc.type & 2)) && write)
602 /* unreadable code segment */
603 if (!fetch && (desc.type & 8) && !(desc.type & 2))
605 lim = desc_limit_scaled(&desc);
606 if ((desc.type & 8) || !(desc.type & 4)) {
607 /* expand-up segment */
608 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
611 /* exapand-down segment */
612 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
614 lim = desc.d ? 0xffffffff : 0xffff;
615 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
618 cpl = ctxt->ops->cpl(ctxt);
621 if (!(desc.type & 8)) {
625 } else if ((desc.type & 8) && !(desc.type & 4)) {
626 /* nonconforming code segment */
629 } else if ((desc.type & 8) && (desc.type & 4)) {
630 /* conforming code segment */
636 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
639 return X86EMUL_CONTINUE;
641 if (addr.seg == VCPU_SREG_SS)
642 return emulate_ss(ctxt, addr.seg);
644 return emulate_gp(ctxt, addr.seg);
647 static int linearize(struct x86_emulate_ctxt *ctxt,
648 struct segmented_address addr,
649 unsigned size, bool write,
652 return __linearize(ctxt, addr, size, write, false, linear);
656 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
657 struct segmented_address addr,
664 rc = linearize(ctxt, addr, size, false, &linear);
665 if (rc != X86EMUL_CONTINUE)
667 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
671 * Fetch the next byte of the instruction being emulated which is pointed to
672 * by ctxt->_eip, then increment ctxt->_eip.
674 * Also prefetch the remaining bytes of the instruction without crossing page
675 * boundary if they are not in fetch_cache yet.
677 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
679 struct fetch_cache *fc = &ctxt->fetch;
683 if (ctxt->_eip == fc->end) {
684 unsigned long linear;
685 struct segmented_address addr = { .seg = VCPU_SREG_CS,
687 cur_size = fc->end - fc->start;
688 size = min(15UL - cur_size,
689 PAGE_SIZE - offset_in_page(ctxt->_eip));
690 rc = __linearize(ctxt, addr, size, false, true, &linear);
691 if (unlikely(rc != X86EMUL_CONTINUE))
693 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
694 size, &ctxt->exception);
695 if (unlikely(rc != X86EMUL_CONTINUE))
699 *dest = fc->data[ctxt->_eip - fc->start];
701 return X86EMUL_CONTINUE;
704 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
705 void *dest, unsigned size)
709 /* x86 instructions are limited to 15 bytes. */
710 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
711 return X86EMUL_UNHANDLEABLE;
713 rc = do_insn_fetch_byte(ctxt, dest++);
714 if (rc != X86EMUL_CONTINUE)
717 return X86EMUL_CONTINUE;
720 /* Fetch next part of the instruction being emulated. */
721 #define insn_fetch(_type, _ctxt) \
722 ({ unsigned long _x; \
723 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
724 if (rc != X86EMUL_CONTINUE) \
729 #define insn_fetch_arr(_arr, _size, _ctxt) \
730 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
731 if (rc != X86EMUL_CONTINUE) \
736 * Given the 'reg' portion of a ModRM byte, and a register block, return a
737 * pointer into the block that addresses the relevant register.
738 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
740 static void *decode_register(u8 modrm_reg, unsigned long *regs,
745 p = ®s[modrm_reg];
746 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
747 p = (unsigned char *)®s[modrm_reg & 3] + 1;
751 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
752 struct segmented_address addr,
753 u16 *size, unsigned long *address, int op_bytes)
760 rc = segmented_read_std(ctxt, addr, size, 2);
761 if (rc != X86EMUL_CONTINUE)
764 rc = segmented_read_std(ctxt, addr, address, op_bytes);
768 static int test_cc(unsigned int condition, unsigned int flags)
772 switch ((condition & 15) >> 1) {
774 rc |= (flags & EFLG_OF);
776 case 1: /* b/c/nae */
777 rc |= (flags & EFLG_CF);
780 rc |= (flags & EFLG_ZF);
783 rc |= (flags & (EFLG_CF|EFLG_ZF));
786 rc |= (flags & EFLG_SF);
789 rc |= (flags & EFLG_PF);
792 rc |= (flags & EFLG_ZF);
795 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
799 /* Odd condition identifiers (lsb == 1) have inverted sense. */
800 return (!!rc ^ (condition & 1));
803 static void fetch_register_operand(struct operand *op)
807 op->val = *(u8 *)op->addr.reg;
810 op->val = *(u16 *)op->addr.reg;
813 op->val = *(u32 *)op->addr.reg;
816 op->val = *(u64 *)op->addr.reg;
821 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
823 ctxt->ops->get_fpu(ctxt);
825 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
826 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
827 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
828 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
829 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
830 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
831 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
832 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
834 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
835 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
836 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
837 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
838 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
839 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
840 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
841 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
845 ctxt->ops->put_fpu(ctxt);
848 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
851 ctxt->ops->get_fpu(ctxt);
853 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
854 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
855 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
856 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
857 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
858 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
859 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
860 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
862 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
863 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
864 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
865 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
866 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
867 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
868 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
869 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
873 ctxt->ops->put_fpu(ctxt);
876 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
880 unsigned reg = ctxt->modrm_reg;
881 int highbyte_regs = ctxt->rex_prefix == 0;
883 if (!(ctxt->d & ModRM))
884 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
890 read_sse_reg(ctxt, &op->vec_val, reg);
895 if ((ctxt->d & ByteOp) && !inhibit_bytereg) {
896 op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
899 op->addr.reg = decode_register(reg, ctxt->regs, 0);
900 op->bytes = ctxt->op_bytes;
902 fetch_register_operand(op);
903 op->orig_val = op->val;
906 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
910 int index_reg = 0, base_reg = 0, scale;
911 int rc = X86EMUL_CONTINUE;
914 if (ctxt->rex_prefix) {
915 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
916 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
917 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
920 ctxt->modrm = insn_fetch(u8, ctxt);
921 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
922 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
923 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
924 ctxt->modrm_seg = VCPU_SREG_DS;
926 if (ctxt->modrm_mod == 3) {
928 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
929 op->addr.reg = decode_register(ctxt->modrm_rm,
930 ctxt->regs, ctxt->d & ByteOp);
934 op->addr.xmm = ctxt->modrm_rm;
935 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
938 fetch_register_operand(op);
944 if (ctxt->ad_bytes == 2) {
945 unsigned bx = ctxt->regs[VCPU_REGS_RBX];
946 unsigned bp = ctxt->regs[VCPU_REGS_RBP];
947 unsigned si = ctxt->regs[VCPU_REGS_RSI];
948 unsigned di = ctxt->regs[VCPU_REGS_RDI];
950 /* 16-bit ModR/M decode. */
951 switch (ctxt->modrm_mod) {
953 if (ctxt->modrm_rm == 6)
954 modrm_ea += insn_fetch(u16, ctxt);
957 modrm_ea += insn_fetch(s8, ctxt);
960 modrm_ea += insn_fetch(u16, ctxt);
963 switch (ctxt->modrm_rm) {
983 if (ctxt->modrm_mod != 0)
990 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
991 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
992 ctxt->modrm_seg = VCPU_SREG_SS;
993 modrm_ea = (u16)modrm_ea;
995 /* 32/64-bit ModR/M decode. */
996 if ((ctxt->modrm_rm & 7) == 4) {
997 sib = insn_fetch(u8, ctxt);
998 index_reg |= (sib >> 3) & 7;
1002 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1003 modrm_ea += insn_fetch(s32, ctxt);
1005 modrm_ea += ctxt->regs[base_reg];
1007 modrm_ea += ctxt->regs[index_reg] << scale;
1008 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1009 if (ctxt->mode == X86EMUL_MODE_PROT64)
1010 ctxt->rip_relative = 1;
1012 modrm_ea += ctxt->regs[ctxt->modrm_rm];
1013 switch (ctxt->modrm_mod) {
1015 if (ctxt->modrm_rm == 5)
1016 modrm_ea += insn_fetch(s32, ctxt);
1019 modrm_ea += insn_fetch(s8, ctxt);
1022 modrm_ea += insn_fetch(s32, ctxt);
1026 op->addr.mem.ea = modrm_ea;
1031 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1034 int rc = X86EMUL_CONTINUE;
1037 switch (ctxt->ad_bytes) {
1039 op->addr.mem.ea = insn_fetch(u16, ctxt);
1042 op->addr.mem.ea = insn_fetch(u32, ctxt);
1045 op->addr.mem.ea = insn_fetch(u64, ctxt);
1052 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1056 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1057 mask = ~(ctxt->dst.bytes * 8 - 1);
1059 if (ctxt->src.bytes == 2)
1060 sv = (s16)ctxt->src.val & (s16)mask;
1061 else if (ctxt->src.bytes == 4)
1062 sv = (s32)ctxt->src.val & (s32)mask;
1064 ctxt->dst.addr.mem.ea += (sv >> 3);
1067 /* only subword offset */
1068 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1071 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1072 unsigned long addr, void *dest, unsigned size)
1075 struct read_cache *mc = &ctxt->mem_read;
1078 int n = min(size, 8u);
1080 if (mc->pos < mc->end)
1083 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1085 if (rc != X86EMUL_CONTINUE)
1090 memcpy(dest, mc->data + mc->pos, n);
1095 return X86EMUL_CONTINUE;
1098 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1099 struct segmented_address addr,
1106 rc = linearize(ctxt, addr, size, false, &linear);
1107 if (rc != X86EMUL_CONTINUE)
1109 return read_emulated(ctxt, linear, data, size);
1112 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1113 struct segmented_address addr,
1120 rc = linearize(ctxt, addr, size, true, &linear);
1121 if (rc != X86EMUL_CONTINUE)
1123 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1127 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1128 struct segmented_address addr,
1129 const void *orig_data, const void *data,
1135 rc = linearize(ctxt, addr, size, true, &linear);
1136 if (rc != X86EMUL_CONTINUE)
1138 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1139 size, &ctxt->exception);
1142 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1143 unsigned int size, unsigned short port,
1146 struct read_cache *rc = &ctxt->io_read;
1148 if (rc->pos == rc->end) { /* refill pio read ahead */
1149 unsigned int in_page, n;
1150 unsigned int count = ctxt->rep_prefix ?
1151 address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
1152 in_page = (ctxt->eflags & EFLG_DF) ?
1153 offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
1154 PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
1155 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1159 rc->pos = rc->end = 0;
1160 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1165 memcpy(dest, rc->data + rc->pos, size);
1170 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1171 u16 selector, struct desc_ptr *dt)
1173 struct x86_emulate_ops *ops = ctxt->ops;
1175 if (selector & 1 << 2) {
1176 struct desc_struct desc;
1179 memset (dt, 0, sizeof *dt);
1180 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1183 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1184 dt->address = get_desc_base(&desc);
1186 ops->get_gdt(ctxt, dt);
1189 /* allowed just for 8 bytes segments */
1190 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1191 u16 selector, struct desc_struct *desc)
1194 u16 index = selector >> 3;
1197 get_descriptor_table_ptr(ctxt, selector, &dt);
1199 if (dt.size < index * 8 + 7)
1200 return emulate_gp(ctxt, selector & 0xfffc);
1202 addr = dt.address + index * 8;
1203 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1207 /* allowed just for 8 bytes segments */
1208 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1209 u16 selector, struct desc_struct *desc)
1212 u16 index = selector >> 3;
1215 get_descriptor_table_ptr(ctxt, selector, &dt);
1217 if (dt.size < index * 8 + 7)
1218 return emulate_gp(ctxt, selector & 0xfffc);
1220 addr = dt.address + index * 8;
1221 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1225 /* Does not support long mode */
1226 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1227 u16 selector, int seg)
1229 struct desc_struct seg_desc;
1231 unsigned err_vec = GP_VECTOR;
1233 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1236 memset(&seg_desc, 0, sizeof seg_desc);
1238 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1239 || ctxt->mode == X86EMUL_MODE_REAL) {
1240 /* set real mode segment descriptor */
1241 set_desc_base(&seg_desc, selector << 4);
1242 set_desc_limit(&seg_desc, 0xffff);
1249 /* NULL selector is not valid for TR, CS and SS */
1250 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1254 /* TR should be in GDT only */
1255 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1258 if (null_selector) /* for NULL selector skip all following checks */
1261 ret = read_segment_descriptor(ctxt, selector, &seg_desc);
1262 if (ret != X86EMUL_CONTINUE)
1265 err_code = selector & 0xfffc;
1266 err_vec = GP_VECTOR;
1268 /* can't load system descriptor into segment selecor */
1269 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1273 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1279 cpl = ctxt->ops->cpl(ctxt);
1284 * segment is not a writable data segment or segment
1285 * selector's RPL != CPL or segment selector's RPL != CPL
1287 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1291 if (!(seg_desc.type & 8))
1294 if (seg_desc.type & 4) {
1300 if (rpl > cpl || dpl != cpl)
1303 /* CS(RPL) <- CPL */
1304 selector = (selector & 0xfffc) | cpl;
1307 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1310 case VCPU_SREG_LDTR:
1311 if (seg_desc.s || seg_desc.type != 2)
1314 default: /* DS, ES, FS, or GS */
1316 * segment is not a data or readable code segment or
1317 * ((segment is a data or nonconforming code segment)
1318 * and (both RPL and CPL > DPL))
1320 if ((seg_desc.type & 0xa) == 0x8 ||
1321 (((seg_desc.type & 0xc) != 0xc) &&
1322 (rpl > dpl && cpl > dpl)))
1328 /* mark segment as accessed */
1330 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1331 if (ret != X86EMUL_CONTINUE)
1335 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1336 return X86EMUL_CONTINUE;
1338 emulate_exception(ctxt, err_vec, err_code, true);
1339 return X86EMUL_PROPAGATE_FAULT;
1342 static void write_register_operand(struct operand *op)
1344 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1345 switch (op->bytes) {
1347 *(u8 *)op->addr.reg = (u8)op->val;
1350 *(u16 *)op->addr.reg = (u16)op->val;
1353 *op->addr.reg = (u32)op->val;
1354 break; /* 64b: zero-extend */
1356 *op->addr.reg = op->val;
1361 static int writeback(struct x86_emulate_ctxt *ctxt)
1365 switch (ctxt->dst.type) {
1367 write_register_operand(&ctxt->dst);
1370 if (ctxt->lock_prefix)
1371 rc = segmented_cmpxchg(ctxt,
1373 &ctxt->dst.orig_val,
1377 rc = segmented_write(ctxt,
1381 if (rc != X86EMUL_CONTINUE)
1385 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1393 return X86EMUL_CONTINUE;
1396 static int em_push(struct x86_emulate_ctxt *ctxt)
1398 struct segmented_address addr;
1400 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], -ctxt->op_bytes);
1401 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1402 addr.seg = VCPU_SREG_SS;
1404 /* Disable writeback. */
1405 ctxt->dst.type = OP_NONE;
1406 return segmented_write(ctxt, addr, &ctxt->src.val, ctxt->op_bytes);
1409 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1410 void *dest, int len)
1413 struct segmented_address addr;
1415 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1416 addr.seg = VCPU_SREG_SS;
1417 rc = segmented_read(ctxt, addr, dest, len);
1418 if (rc != X86EMUL_CONTINUE)
1421 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], len);
1425 static int em_pop(struct x86_emulate_ctxt *ctxt)
1427 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1430 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1431 void *dest, int len)
1434 unsigned long val, change_mask;
1435 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1436 int cpl = ctxt->ops->cpl(ctxt);
1438 rc = emulate_pop(ctxt, &val, len);
1439 if (rc != X86EMUL_CONTINUE)
1442 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1443 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1445 switch(ctxt->mode) {
1446 case X86EMUL_MODE_PROT64:
1447 case X86EMUL_MODE_PROT32:
1448 case X86EMUL_MODE_PROT16:
1450 change_mask |= EFLG_IOPL;
1452 change_mask |= EFLG_IF;
1454 case X86EMUL_MODE_VM86:
1456 return emulate_gp(ctxt, 0);
1457 change_mask |= EFLG_IF;
1459 default: /* real mode */
1460 change_mask |= (EFLG_IOPL | EFLG_IF);
1464 *(unsigned long *)dest =
1465 (ctxt->eflags & ~change_mask) | (val & change_mask);
1470 static int em_popf(struct x86_emulate_ctxt *ctxt)
1472 ctxt->dst.type = OP_REG;
1473 ctxt->dst.addr.reg = &ctxt->eflags;
1474 ctxt->dst.bytes = ctxt->op_bytes;
1475 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1478 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1480 int seg = ctxt->src2.val;
1482 ctxt->src.val = get_segment_selector(ctxt, seg);
1484 return em_push(ctxt);
1487 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1489 int seg = ctxt->src2.val;
1490 unsigned long selector;
1493 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1494 if (rc != X86EMUL_CONTINUE)
1497 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1501 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1503 unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
1504 int rc = X86EMUL_CONTINUE;
1505 int reg = VCPU_REGS_RAX;
1507 while (reg <= VCPU_REGS_RDI) {
1508 (reg == VCPU_REGS_RSP) ?
1509 (ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
1512 if (rc != X86EMUL_CONTINUE)
1521 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1523 ctxt->src.val = (unsigned long)ctxt->eflags;
1524 return em_push(ctxt);
1527 static int em_popa(struct x86_emulate_ctxt *ctxt)
1529 int rc = X86EMUL_CONTINUE;
1530 int reg = VCPU_REGS_RDI;
1532 while (reg >= VCPU_REGS_RAX) {
1533 if (reg == VCPU_REGS_RSP) {
1534 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP],
1539 rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
1540 if (rc != X86EMUL_CONTINUE)
1547 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1549 struct x86_emulate_ops *ops = ctxt->ops;
1556 /* TODO: Add limit checks */
1557 ctxt->src.val = ctxt->eflags;
1559 if (rc != X86EMUL_CONTINUE)
1562 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1564 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1566 if (rc != X86EMUL_CONTINUE)
1569 ctxt->src.val = ctxt->_eip;
1571 if (rc != X86EMUL_CONTINUE)
1574 ops->get_idt(ctxt, &dt);
1576 eip_addr = dt.address + (irq << 2);
1577 cs_addr = dt.address + (irq << 2) + 2;
1579 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1580 if (rc != X86EMUL_CONTINUE)
1583 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1584 if (rc != X86EMUL_CONTINUE)
1587 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1588 if (rc != X86EMUL_CONTINUE)
1596 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1598 switch(ctxt->mode) {
1599 case X86EMUL_MODE_REAL:
1600 return emulate_int_real(ctxt, irq);
1601 case X86EMUL_MODE_VM86:
1602 case X86EMUL_MODE_PROT16:
1603 case X86EMUL_MODE_PROT32:
1604 case X86EMUL_MODE_PROT64:
1606 /* Protected mode interrupts unimplemented yet */
1607 return X86EMUL_UNHANDLEABLE;
1611 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1613 int rc = X86EMUL_CONTINUE;
1614 unsigned long temp_eip = 0;
1615 unsigned long temp_eflags = 0;
1616 unsigned long cs = 0;
1617 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1618 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1619 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1620 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1622 /* TODO: Add stack limit check */
1624 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1626 if (rc != X86EMUL_CONTINUE)
1629 if (temp_eip & ~0xffff)
1630 return emulate_gp(ctxt, 0);
1632 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1634 if (rc != X86EMUL_CONTINUE)
1637 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1639 if (rc != X86EMUL_CONTINUE)
1642 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1644 if (rc != X86EMUL_CONTINUE)
1647 ctxt->_eip = temp_eip;
1650 if (ctxt->op_bytes == 4)
1651 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1652 else if (ctxt->op_bytes == 2) {
1653 ctxt->eflags &= ~0xffff;
1654 ctxt->eflags |= temp_eflags;
1657 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1658 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1663 static int em_iret(struct x86_emulate_ctxt *ctxt)
1665 switch(ctxt->mode) {
1666 case X86EMUL_MODE_REAL:
1667 return emulate_iret_real(ctxt);
1668 case X86EMUL_MODE_VM86:
1669 case X86EMUL_MODE_PROT16:
1670 case X86EMUL_MODE_PROT32:
1671 case X86EMUL_MODE_PROT64:
1673 /* iret from protected mode unimplemented yet */
1674 return X86EMUL_UNHANDLEABLE;
1678 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1683 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1685 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1686 if (rc != X86EMUL_CONTINUE)
1690 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1691 return X86EMUL_CONTINUE;
1694 static int em_grp1a(struct x86_emulate_ctxt *ctxt)
1696 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->dst.bytes);
1699 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1701 switch (ctxt->modrm_reg) {
1703 emulate_2op_SrcB(ctxt, "rol");
1706 emulate_2op_SrcB(ctxt, "ror");
1709 emulate_2op_SrcB(ctxt, "rcl");
1712 emulate_2op_SrcB(ctxt, "rcr");
1714 case 4: /* sal/shl */
1715 case 6: /* sal/shl */
1716 emulate_2op_SrcB(ctxt, "sal");
1719 emulate_2op_SrcB(ctxt, "shr");
1722 emulate_2op_SrcB(ctxt, "sar");
1725 return X86EMUL_CONTINUE;
1728 static int em_not(struct x86_emulate_ctxt *ctxt)
1730 ctxt->dst.val = ~ctxt->dst.val;
1731 return X86EMUL_CONTINUE;
1734 static int em_neg(struct x86_emulate_ctxt *ctxt)
1736 emulate_1op(ctxt, "neg");
1737 return X86EMUL_CONTINUE;
1740 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
1744 emulate_1op_rax_rdx(ctxt, "mul", ex);
1745 return X86EMUL_CONTINUE;
1748 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
1752 emulate_1op_rax_rdx(ctxt, "imul", ex);
1753 return X86EMUL_CONTINUE;
1756 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1760 emulate_1op_rax_rdx(ctxt, "div", de);
1762 return emulate_de(ctxt);
1763 return X86EMUL_CONTINUE;
1766 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
1770 emulate_1op_rax_rdx(ctxt, "idiv", de);
1772 return emulate_de(ctxt);
1773 return X86EMUL_CONTINUE;
1776 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1778 int rc = X86EMUL_CONTINUE;
1780 switch (ctxt->modrm_reg) {
1782 emulate_1op(ctxt, "inc");
1785 emulate_1op(ctxt, "dec");
1787 case 2: /* call near abs */ {
1789 old_eip = ctxt->_eip;
1790 ctxt->_eip = ctxt->src.val;
1791 ctxt->src.val = old_eip;
1795 case 4: /* jmp abs */
1796 ctxt->_eip = ctxt->src.val;
1798 case 5: /* jmp far */
1799 rc = em_jmp_far(ctxt);
1808 static int em_grp9(struct x86_emulate_ctxt *ctxt)
1810 u64 old = ctxt->dst.orig_val64;
1812 if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
1813 ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
1814 ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1815 ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1816 ctxt->eflags &= ~EFLG_ZF;
1818 ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
1819 (u32) ctxt->regs[VCPU_REGS_RBX];
1821 ctxt->eflags |= EFLG_ZF;
1823 return X86EMUL_CONTINUE;
1826 static int em_ret(struct x86_emulate_ctxt *ctxt)
1828 ctxt->dst.type = OP_REG;
1829 ctxt->dst.addr.reg = &ctxt->_eip;
1830 ctxt->dst.bytes = ctxt->op_bytes;
1831 return em_pop(ctxt);
1834 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
1838 int cpl = ctxt->ops->cpl(ctxt);
1840 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
1841 if (rc != X86EMUL_CONTINUE)
1843 if (ctxt->op_bytes == 4)
1844 ctxt->_eip = (u32)ctxt->_eip;
1845 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1846 if (rc != X86EMUL_CONTINUE)
1848 /* Outer-privilege level return is not implemented */
1849 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
1850 return X86EMUL_UNHANDLEABLE;
1851 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1855 static int em_lseg(struct x86_emulate_ctxt *ctxt)
1857 int seg = ctxt->src2.val;
1861 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1863 rc = load_segment_descriptor(ctxt, sel, seg);
1864 if (rc != X86EMUL_CONTINUE)
1867 ctxt->dst.val = ctxt->src.val;
1872 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1873 struct desc_struct *cs, struct desc_struct *ss)
1877 memset(cs, 0, sizeof(struct desc_struct));
1878 ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1879 memset(ss, 0, sizeof(struct desc_struct));
1881 cs->l = 0; /* will be adjusted later */
1882 set_desc_base(cs, 0); /* flat segment */
1883 cs->g = 1; /* 4kb granularity */
1884 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1885 cs->type = 0x0b; /* Read, Execute, Accessed */
1887 cs->dpl = 0; /* will be adjusted later */
1891 set_desc_base(ss, 0); /* flat segment */
1892 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1893 ss->g = 1; /* 4kb granularity */
1895 ss->type = 0x03; /* Read/Write, Accessed */
1896 ss->d = 1; /* 32bit stack segment */
1901 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
1903 struct x86_emulate_ops *ops = ctxt->ops;
1904 u32 eax, ebx, ecx, edx;
1907 * syscall should always be enabled in longmode - so only become
1908 * vendor specific (cpuid) if other modes are active...
1910 if (ctxt->mode == X86EMUL_MODE_PROT64)
1915 if (ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx)) {
1917 * Intel ("GenuineIntel")
1918 * remark: Intel CPUs only support "syscall" in 64bit
1919 * longmode. Also an 64bit guest with a
1920 * 32bit compat-app running will #UD !! While this
1921 * behaviour can be fixed (by emulating) into AMD
1922 * response - CPUs of AMD can't behave like Intel.
1924 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
1925 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
1926 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
1929 /* AMD ("AuthenticAMD") */
1930 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
1931 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
1932 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
1935 /* AMD ("AMDisbetter!") */
1936 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
1937 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
1938 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
1942 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
1946 static int em_syscall(struct x86_emulate_ctxt *ctxt)
1948 struct x86_emulate_ops *ops = ctxt->ops;
1949 struct desc_struct cs, ss;
1954 /* syscall is not available in real mode */
1955 if (ctxt->mode == X86EMUL_MODE_REAL ||
1956 ctxt->mode == X86EMUL_MODE_VM86)
1957 return emulate_ud(ctxt);
1959 if (!(em_syscall_is_enabled(ctxt)))
1960 return emulate_ud(ctxt);
1962 ops->get_msr(ctxt, MSR_EFER, &efer);
1963 setup_syscalls_segments(ctxt, &cs, &ss);
1965 if (!(efer & EFER_SCE))
1966 return emulate_ud(ctxt);
1968 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1970 cs_sel = (u16)(msr_data & 0xfffc);
1971 ss_sel = (u16)(msr_data + 8);
1973 if (efer & EFER_LMA) {
1977 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1978 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1980 ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
1981 if (efer & EFER_LMA) {
1982 #ifdef CONFIG_X86_64
1983 ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1986 ctxt->mode == X86EMUL_MODE_PROT64 ?
1987 MSR_LSTAR : MSR_CSTAR, &msr_data);
1988 ctxt->_eip = msr_data;
1990 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1991 ctxt->eflags &= ~(msr_data | EFLG_RF);
1995 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1996 ctxt->_eip = (u32)msr_data;
1998 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2001 return X86EMUL_CONTINUE;
2004 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2006 struct x86_emulate_ops *ops = ctxt->ops;
2007 struct desc_struct cs, ss;
2012 ops->get_msr(ctxt, MSR_EFER, &efer);
2013 /* inject #GP if in real mode */
2014 if (ctxt->mode == X86EMUL_MODE_REAL)
2015 return emulate_gp(ctxt, 0);
2017 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2018 * Therefore, we inject an #UD.
2020 if (ctxt->mode == X86EMUL_MODE_PROT64)
2021 return emulate_ud(ctxt);
2023 setup_syscalls_segments(ctxt, &cs, &ss);
2025 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2026 switch (ctxt->mode) {
2027 case X86EMUL_MODE_PROT32:
2028 if ((msr_data & 0xfffc) == 0x0)
2029 return emulate_gp(ctxt, 0);
2031 case X86EMUL_MODE_PROT64:
2032 if (msr_data == 0x0)
2033 return emulate_gp(ctxt, 0);
2037 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2038 cs_sel = (u16)msr_data;
2039 cs_sel &= ~SELECTOR_RPL_MASK;
2040 ss_sel = cs_sel + 8;
2041 ss_sel &= ~SELECTOR_RPL_MASK;
2042 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2047 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2048 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2050 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2051 ctxt->_eip = msr_data;
2053 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2054 ctxt->regs[VCPU_REGS_RSP] = msr_data;
2056 return X86EMUL_CONTINUE;
2059 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2061 struct x86_emulate_ops *ops = ctxt->ops;
2062 struct desc_struct cs, ss;
2065 u16 cs_sel = 0, ss_sel = 0;
2067 /* inject #GP if in real mode or Virtual 8086 mode */
2068 if (ctxt->mode == X86EMUL_MODE_REAL ||
2069 ctxt->mode == X86EMUL_MODE_VM86)
2070 return emulate_gp(ctxt, 0);
2072 setup_syscalls_segments(ctxt, &cs, &ss);
2074 if ((ctxt->rex_prefix & 0x8) != 0x0)
2075 usermode = X86EMUL_MODE_PROT64;
2077 usermode = X86EMUL_MODE_PROT32;
2081 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2083 case X86EMUL_MODE_PROT32:
2084 cs_sel = (u16)(msr_data + 16);
2085 if ((msr_data & 0xfffc) == 0x0)
2086 return emulate_gp(ctxt, 0);
2087 ss_sel = (u16)(msr_data + 24);
2089 case X86EMUL_MODE_PROT64:
2090 cs_sel = (u16)(msr_data + 32);
2091 if (msr_data == 0x0)
2092 return emulate_gp(ctxt, 0);
2093 ss_sel = cs_sel + 8;
2098 cs_sel |= SELECTOR_RPL_MASK;
2099 ss_sel |= SELECTOR_RPL_MASK;
2101 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2102 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2104 ctxt->_eip = ctxt->regs[VCPU_REGS_RDX];
2105 ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX];
2107 return X86EMUL_CONTINUE;
2110 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2113 if (ctxt->mode == X86EMUL_MODE_REAL)
2115 if (ctxt->mode == X86EMUL_MODE_VM86)
2117 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2118 return ctxt->ops->cpl(ctxt) > iopl;
2121 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2124 struct x86_emulate_ops *ops = ctxt->ops;
2125 struct desc_struct tr_seg;
2128 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2129 unsigned mask = (1 << len) - 1;
2132 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2135 if (desc_limit_scaled(&tr_seg) < 103)
2137 base = get_desc_base(&tr_seg);
2138 #ifdef CONFIG_X86_64
2139 base |= ((u64)base3) << 32;
2141 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2142 if (r != X86EMUL_CONTINUE)
2144 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2146 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2147 if (r != X86EMUL_CONTINUE)
2149 if ((perm >> bit_idx) & mask)
2154 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2160 if (emulator_bad_iopl(ctxt))
2161 if (!emulator_io_port_access_allowed(ctxt, port, len))
2164 ctxt->perm_ok = true;
2169 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2170 struct tss_segment_16 *tss)
2172 tss->ip = ctxt->_eip;
2173 tss->flag = ctxt->eflags;
2174 tss->ax = ctxt->regs[VCPU_REGS_RAX];
2175 tss->cx = ctxt->regs[VCPU_REGS_RCX];
2176 tss->dx = ctxt->regs[VCPU_REGS_RDX];
2177 tss->bx = ctxt->regs[VCPU_REGS_RBX];
2178 tss->sp = ctxt->regs[VCPU_REGS_RSP];
2179 tss->bp = ctxt->regs[VCPU_REGS_RBP];
2180 tss->si = ctxt->regs[VCPU_REGS_RSI];
2181 tss->di = ctxt->regs[VCPU_REGS_RDI];
2183 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2184 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2185 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2186 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2187 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2190 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2191 struct tss_segment_16 *tss)
2195 ctxt->_eip = tss->ip;
2196 ctxt->eflags = tss->flag | 2;
2197 ctxt->regs[VCPU_REGS_RAX] = tss->ax;
2198 ctxt->regs[VCPU_REGS_RCX] = tss->cx;
2199 ctxt->regs[VCPU_REGS_RDX] = tss->dx;
2200 ctxt->regs[VCPU_REGS_RBX] = tss->bx;
2201 ctxt->regs[VCPU_REGS_RSP] = tss->sp;
2202 ctxt->regs[VCPU_REGS_RBP] = tss->bp;
2203 ctxt->regs[VCPU_REGS_RSI] = tss->si;
2204 ctxt->regs[VCPU_REGS_RDI] = tss->di;
2207 * SDM says that segment selectors are loaded before segment
2210 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2211 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2212 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2213 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2214 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2217 * Now load segment descriptors. If fault happenes at this stage
2218 * it is handled in a context of new task
2220 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2221 if (ret != X86EMUL_CONTINUE)
2223 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2224 if (ret != X86EMUL_CONTINUE)
2226 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2227 if (ret != X86EMUL_CONTINUE)
2229 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2230 if (ret != X86EMUL_CONTINUE)
2232 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2233 if (ret != X86EMUL_CONTINUE)
2236 return X86EMUL_CONTINUE;
2239 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2240 u16 tss_selector, u16 old_tss_sel,
2241 ulong old_tss_base, struct desc_struct *new_desc)
2243 struct x86_emulate_ops *ops = ctxt->ops;
2244 struct tss_segment_16 tss_seg;
2246 u32 new_tss_base = get_desc_base(new_desc);
2248 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2250 if (ret != X86EMUL_CONTINUE)
2251 /* FIXME: need to provide precise fault address */
2254 save_state_to_tss16(ctxt, &tss_seg);
2256 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2258 if (ret != X86EMUL_CONTINUE)
2259 /* FIXME: need to provide precise fault address */
2262 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2264 if (ret != X86EMUL_CONTINUE)
2265 /* FIXME: need to provide precise fault address */
2268 if (old_tss_sel != 0xffff) {
2269 tss_seg.prev_task_link = old_tss_sel;
2271 ret = ops->write_std(ctxt, new_tss_base,
2272 &tss_seg.prev_task_link,
2273 sizeof tss_seg.prev_task_link,
2275 if (ret != X86EMUL_CONTINUE)
2276 /* FIXME: need to provide precise fault address */
2280 return load_state_from_tss16(ctxt, &tss_seg);
2283 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2284 struct tss_segment_32 *tss)
2286 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2287 tss->eip = ctxt->_eip;
2288 tss->eflags = ctxt->eflags;
2289 tss->eax = ctxt->regs[VCPU_REGS_RAX];
2290 tss->ecx = ctxt->regs[VCPU_REGS_RCX];
2291 tss->edx = ctxt->regs[VCPU_REGS_RDX];
2292 tss->ebx = ctxt->regs[VCPU_REGS_RBX];
2293 tss->esp = ctxt->regs[VCPU_REGS_RSP];
2294 tss->ebp = ctxt->regs[VCPU_REGS_RBP];
2295 tss->esi = ctxt->regs[VCPU_REGS_RSI];
2296 tss->edi = ctxt->regs[VCPU_REGS_RDI];
2298 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2299 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2300 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2301 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2302 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2303 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2304 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2307 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2308 struct tss_segment_32 *tss)
2312 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2313 return emulate_gp(ctxt, 0);
2314 ctxt->_eip = tss->eip;
2315 ctxt->eflags = tss->eflags | 2;
2316 ctxt->regs[VCPU_REGS_RAX] = tss->eax;
2317 ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
2318 ctxt->regs[VCPU_REGS_RDX] = tss->edx;
2319 ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
2320 ctxt->regs[VCPU_REGS_RSP] = tss->esp;
2321 ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
2322 ctxt->regs[VCPU_REGS_RSI] = tss->esi;
2323 ctxt->regs[VCPU_REGS_RDI] = tss->edi;
2326 * SDM says that segment selectors are loaded before segment
2329 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2330 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2331 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2332 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2333 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2334 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2335 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2338 * Now load segment descriptors. If fault happenes at this stage
2339 * it is handled in a context of new task
2341 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2342 if (ret != X86EMUL_CONTINUE)
2344 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2345 if (ret != X86EMUL_CONTINUE)
2347 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2348 if (ret != X86EMUL_CONTINUE)
2350 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2351 if (ret != X86EMUL_CONTINUE)
2353 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2354 if (ret != X86EMUL_CONTINUE)
2356 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2357 if (ret != X86EMUL_CONTINUE)
2359 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2360 if (ret != X86EMUL_CONTINUE)
2363 return X86EMUL_CONTINUE;
2366 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2367 u16 tss_selector, u16 old_tss_sel,
2368 ulong old_tss_base, struct desc_struct *new_desc)
2370 struct x86_emulate_ops *ops = ctxt->ops;
2371 struct tss_segment_32 tss_seg;
2373 u32 new_tss_base = get_desc_base(new_desc);
2375 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2377 if (ret != X86EMUL_CONTINUE)
2378 /* FIXME: need to provide precise fault address */
2381 save_state_to_tss32(ctxt, &tss_seg);
2383 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2385 if (ret != X86EMUL_CONTINUE)
2386 /* FIXME: need to provide precise fault address */
2389 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2391 if (ret != X86EMUL_CONTINUE)
2392 /* FIXME: need to provide precise fault address */
2395 if (old_tss_sel != 0xffff) {
2396 tss_seg.prev_task_link = old_tss_sel;
2398 ret = ops->write_std(ctxt, new_tss_base,
2399 &tss_seg.prev_task_link,
2400 sizeof tss_seg.prev_task_link,
2402 if (ret != X86EMUL_CONTINUE)
2403 /* FIXME: need to provide precise fault address */
2407 return load_state_from_tss32(ctxt, &tss_seg);
2410 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2411 u16 tss_selector, int reason,
2412 bool has_error_code, u32 error_code)
2414 struct x86_emulate_ops *ops = ctxt->ops;
2415 struct desc_struct curr_tss_desc, next_tss_desc;
2417 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2418 ulong old_tss_base =
2419 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2422 /* FIXME: old_tss_base == ~0 ? */
2424 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2425 if (ret != X86EMUL_CONTINUE)
2427 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2428 if (ret != X86EMUL_CONTINUE)
2431 /* FIXME: check that next_tss_desc is tss */
2433 if (reason != TASK_SWITCH_IRET) {
2434 if ((tss_selector & 3) > next_tss_desc.dpl ||
2435 ops->cpl(ctxt) > next_tss_desc.dpl)
2436 return emulate_gp(ctxt, 0);
2439 desc_limit = desc_limit_scaled(&next_tss_desc);
2440 if (!next_tss_desc.p ||
2441 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2442 desc_limit < 0x2b)) {
2443 emulate_ts(ctxt, tss_selector & 0xfffc);
2444 return X86EMUL_PROPAGATE_FAULT;
2447 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2448 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2449 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2452 if (reason == TASK_SWITCH_IRET)
2453 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2455 /* set back link to prev task only if NT bit is set in eflags
2456 note that old_tss_sel is not used afetr this point */
2457 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2458 old_tss_sel = 0xffff;
2460 if (next_tss_desc.type & 8)
2461 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2462 old_tss_base, &next_tss_desc);
2464 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2465 old_tss_base, &next_tss_desc);
2466 if (ret != X86EMUL_CONTINUE)
2469 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2470 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2472 if (reason != TASK_SWITCH_IRET) {
2473 next_tss_desc.type |= (1 << 1); /* set busy flag */
2474 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2477 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2478 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2480 if (has_error_code) {
2481 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2482 ctxt->lock_prefix = 0;
2483 ctxt->src.val = (unsigned long) error_code;
2484 ret = em_push(ctxt);
2490 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2491 u16 tss_selector, int reason,
2492 bool has_error_code, u32 error_code)
2496 ctxt->_eip = ctxt->eip;
2497 ctxt->dst.type = OP_NONE;
2499 rc = emulator_do_task_switch(ctxt, tss_selector, reason,
2500 has_error_code, error_code);
2502 if (rc == X86EMUL_CONTINUE)
2503 ctxt->eip = ctxt->_eip;
2505 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2508 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2509 int reg, struct operand *op)
2511 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2513 register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
2514 op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
2515 op->addr.mem.seg = seg;
2518 static int em_das(struct x86_emulate_ctxt *ctxt)
2521 bool af, cf, old_cf;
2523 cf = ctxt->eflags & X86_EFLAGS_CF;
2529 af = ctxt->eflags & X86_EFLAGS_AF;
2530 if ((al & 0x0f) > 9 || af) {
2532 cf = old_cf | (al >= 250);
2537 if (old_al > 0x99 || old_cf) {
2543 /* Set PF, ZF, SF */
2544 ctxt->src.type = OP_IMM;
2546 ctxt->src.bytes = 1;
2547 emulate_2op_SrcV(ctxt, "or");
2548 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2550 ctxt->eflags |= X86_EFLAGS_CF;
2552 ctxt->eflags |= X86_EFLAGS_AF;
2553 return X86EMUL_CONTINUE;
2556 static int em_call(struct x86_emulate_ctxt *ctxt)
2558 long rel = ctxt->src.val;
2560 ctxt->src.val = (unsigned long)ctxt->_eip;
2562 return em_push(ctxt);
2565 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2571 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2572 old_eip = ctxt->_eip;
2574 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2575 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2576 return X86EMUL_CONTINUE;
2579 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2581 ctxt->src.val = old_cs;
2583 if (rc != X86EMUL_CONTINUE)
2586 ctxt->src.val = old_eip;
2587 return em_push(ctxt);
2590 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2594 ctxt->dst.type = OP_REG;
2595 ctxt->dst.addr.reg = &ctxt->_eip;
2596 ctxt->dst.bytes = ctxt->op_bytes;
2597 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2598 if (rc != X86EMUL_CONTINUE)
2600 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], ctxt->src.val);
2601 return X86EMUL_CONTINUE;
2604 static int em_add(struct x86_emulate_ctxt *ctxt)
2606 emulate_2op_SrcV(ctxt, "add");
2607 return X86EMUL_CONTINUE;
2610 static int em_or(struct x86_emulate_ctxt *ctxt)
2612 emulate_2op_SrcV(ctxt, "or");
2613 return X86EMUL_CONTINUE;
2616 static int em_adc(struct x86_emulate_ctxt *ctxt)
2618 emulate_2op_SrcV(ctxt, "adc");
2619 return X86EMUL_CONTINUE;
2622 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2624 emulate_2op_SrcV(ctxt, "sbb");
2625 return X86EMUL_CONTINUE;
2628 static int em_and(struct x86_emulate_ctxt *ctxt)
2630 emulate_2op_SrcV(ctxt, "and");
2631 return X86EMUL_CONTINUE;
2634 static int em_sub(struct x86_emulate_ctxt *ctxt)
2636 emulate_2op_SrcV(ctxt, "sub");
2637 return X86EMUL_CONTINUE;
2640 static int em_xor(struct x86_emulate_ctxt *ctxt)
2642 emulate_2op_SrcV(ctxt, "xor");
2643 return X86EMUL_CONTINUE;
2646 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2648 emulate_2op_SrcV(ctxt, "cmp");
2649 /* Disable writeback. */
2650 ctxt->dst.type = OP_NONE;
2651 return X86EMUL_CONTINUE;
2654 static int em_test(struct x86_emulate_ctxt *ctxt)
2656 emulate_2op_SrcV(ctxt, "test");
2657 /* Disable writeback. */
2658 ctxt->dst.type = OP_NONE;
2659 return X86EMUL_CONTINUE;
2662 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2664 /* Write back the register source. */
2665 ctxt->src.val = ctxt->dst.val;
2666 write_register_operand(&ctxt->src);
2668 /* Write back the memory destination with implicit LOCK prefix. */
2669 ctxt->dst.val = ctxt->src.orig_val;
2670 ctxt->lock_prefix = 1;
2671 return X86EMUL_CONTINUE;
2674 static int em_imul(struct x86_emulate_ctxt *ctxt)
2676 emulate_2op_SrcV_nobyte(ctxt, "imul");
2677 return X86EMUL_CONTINUE;
2680 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2682 ctxt->dst.val = ctxt->src2.val;
2683 return em_imul(ctxt);
2686 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2688 ctxt->dst.type = OP_REG;
2689 ctxt->dst.bytes = ctxt->src.bytes;
2690 ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
2691 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2693 return X86EMUL_CONTINUE;
2696 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2700 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2701 ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
2702 ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
2703 return X86EMUL_CONTINUE;
2706 static int em_mov(struct x86_emulate_ctxt *ctxt)
2708 ctxt->dst.val = ctxt->src.val;
2709 return X86EMUL_CONTINUE;
2712 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
2714 if (ctxt->modrm_reg > VCPU_SREG_GS)
2715 return emulate_ud(ctxt);
2717 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
2718 return X86EMUL_CONTINUE;
2721 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
2723 u16 sel = ctxt->src.val;
2725 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
2726 return emulate_ud(ctxt);
2728 if (ctxt->modrm_reg == VCPU_SREG_SS)
2729 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2731 /* Disable writeback. */
2732 ctxt->dst.type = OP_NONE;
2733 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
2736 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2738 memcpy(&ctxt->dst.vec_val, &ctxt->src.vec_val, ctxt->op_bytes);
2739 return X86EMUL_CONTINUE;
2742 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2747 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
2748 if (rc == X86EMUL_CONTINUE)
2749 ctxt->ops->invlpg(ctxt, linear);
2750 /* Disable writeback. */
2751 ctxt->dst.type = OP_NONE;
2752 return X86EMUL_CONTINUE;
2755 static int em_clts(struct x86_emulate_ctxt *ctxt)
2759 cr0 = ctxt->ops->get_cr(ctxt, 0);
2761 ctxt->ops->set_cr(ctxt, 0, cr0);
2762 return X86EMUL_CONTINUE;
2765 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
2769 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
2770 return X86EMUL_UNHANDLEABLE;
2772 rc = ctxt->ops->fix_hypercall(ctxt);
2773 if (rc != X86EMUL_CONTINUE)
2776 /* Let the processor re-execute the fixed hypercall */
2777 ctxt->_eip = ctxt->eip;
2778 /* Disable writeback. */
2779 ctxt->dst.type = OP_NONE;
2780 return X86EMUL_CONTINUE;
2783 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
2785 struct desc_ptr desc_ptr;
2788 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2789 &desc_ptr.size, &desc_ptr.address,
2791 if (rc != X86EMUL_CONTINUE)
2793 ctxt->ops->set_gdt(ctxt, &desc_ptr);
2794 /* Disable writeback. */
2795 ctxt->dst.type = OP_NONE;
2796 return X86EMUL_CONTINUE;
2799 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2803 rc = ctxt->ops->fix_hypercall(ctxt);
2805 /* Disable writeback. */
2806 ctxt->dst.type = OP_NONE;
2810 static int em_lidt(struct x86_emulate_ctxt *ctxt)
2812 struct desc_ptr desc_ptr;
2815 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2816 &desc_ptr.size, &desc_ptr.address,
2818 if (rc != X86EMUL_CONTINUE)
2820 ctxt->ops->set_idt(ctxt, &desc_ptr);
2821 /* Disable writeback. */
2822 ctxt->dst.type = OP_NONE;
2823 return X86EMUL_CONTINUE;
2826 static int em_smsw(struct x86_emulate_ctxt *ctxt)
2828 ctxt->dst.bytes = 2;
2829 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
2830 return X86EMUL_CONTINUE;
2833 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
2835 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2836 | (ctxt->src.val & 0x0f));
2837 ctxt->dst.type = OP_NONE;
2838 return X86EMUL_CONTINUE;
2841 static int em_loop(struct x86_emulate_ctxt *ctxt)
2843 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
2844 if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
2845 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
2846 jmp_rel(ctxt, ctxt->src.val);
2848 return X86EMUL_CONTINUE;
2851 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
2853 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
2854 jmp_rel(ctxt, ctxt->src.val);
2856 return X86EMUL_CONTINUE;
2859 static int em_cli(struct x86_emulate_ctxt *ctxt)
2861 if (emulator_bad_iopl(ctxt))
2862 return emulate_gp(ctxt, 0);
2864 ctxt->eflags &= ~X86_EFLAGS_IF;
2865 return X86EMUL_CONTINUE;
2868 static int em_sti(struct x86_emulate_ctxt *ctxt)
2870 if (emulator_bad_iopl(ctxt))
2871 return emulate_gp(ctxt, 0);
2873 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
2874 ctxt->eflags |= X86_EFLAGS_IF;
2875 return X86EMUL_CONTINUE;
2878 static bool valid_cr(int nr)
2890 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2892 if (!valid_cr(ctxt->modrm_reg))
2893 return emulate_ud(ctxt);
2895 return X86EMUL_CONTINUE;
2898 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2900 u64 new_val = ctxt->src.val64;
2901 int cr = ctxt->modrm_reg;
2904 static u64 cr_reserved_bits[] = {
2905 0xffffffff00000000ULL,
2906 0, 0, 0, /* CR3 checked later */
2913 return emulate_ud(ctxt);
2915 if (new_val & cr_reserved_bits[cr])
2916 return emulate_gp(ctxt, 0);
2921 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2922 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2923 return emulate_gp(ctxt, 0);
2925 cr4 = ctxt->ops->get_cr(ctxt, 4);
2926 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2928 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2929 !(cr4 & X86_CR4_PAE))
2930 return emulate_gp(ctxt, 0);
2937 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2938 if (efer & EFER_LMA)
2939 rsvd = CR3_L_MODE_RESERVED_BITS;
2940 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
2941 rsvd = CR3_PAE_RESERVED_BITS;
2942 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
2943 rsvd = CR3_NONPAE_RESERVED_BITS;
2946 return emulate_gp(ctxt, 0);
2953 cr4 = ctxt->ops->get_cr(ctxt, 4);
2954 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2956 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2957 return emulate_gp(ctxt, 0);
2963 return X86EMUL_CONTINUE;
2966 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2970 ctxt->ops->get_dr(ctxt, 7, &dr7);
2972 /* Check if DR7.Global_Enable is set */
2973 return dr7 & (1 << 13);
2976 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2978 int dr = ctxt->modrm_reg;
2982 return emulate_ud(ctxt);
2984 cr4 = ctxt->ops->get_cr(ctxt, 4);
2985 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2986 return emulate_ud(ctxt);
2988 if (check_dr7_gd(ctxt))
2989 return emulate_db(ctxt);
2991 return X86EMUL_CONTINUE;
2994 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2996 u64 new_val = ctxt->src.val64;
2997 int dr = ctxt->modrm_reg;
2999 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3000 return emulate_gp(ctxt, 0);
3002 return check_dr_read(ctxt);
3005 static int check_svme(struct x86_emulate_ctxt *ctxt)
3009 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3011 if (!(efer & EFER_SVME))
3012 return emulate_ud(ctxt);
3014 return X86EMUL_CONTINUE;
3017 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3019 u64 rax = ctxt->regs[VCPU_REGS_RAX];
3021 /* Valid physical address? */
3022 if (rax & 0xffff000000000000ULL)
3023 return emulate_gp(ctxt, 0);
3025 return check_svme(ctxt);
3028 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3030 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3032 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3033 return emulate_ud(ctxt);
3035 return X86EMUL_CONTINUE;
3038 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3040 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3041 u64 rcx = ctxt->regs[VCPU_REGS_RCX];
3043 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3045 return emulate_gp(ctxt, 0);
3047 return X86EMUL_CONTINUE;
3050 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3052 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3053 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3054 return emulate_gp(ctxt, 0);
3056 return X86EMUL_CONTINUE;
3059 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3061 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3062 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3063 return emulate_gp(ctxt, 0);
3065 return X86EMUL_CONTINUE;
3068 #define D(_y) { .flags = (_y) }
3069 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3070 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3071 .check_perm = (_p) }
3073 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3074 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
3075 #define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
3076 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3077 #define II(_f, _e, _i) \
3078 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3079 #define IIP(_f, _e, _i, _p) \
3080 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3081 .check_perm = (_p) }
3082 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3084 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3085 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3086 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3088 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3089 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3090 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3092 static struct opcode group7_rm1[] = {
3093 DI(SrcNone | ModRM | Priv, monitor),
3094 DI(SrcNone | ModRM | Priv, mwait),
3098 static struct opcode group7_rm3[] = {
3099 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
3100 II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
3101 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
3102 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
3103 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
3104 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
3105 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
3106 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
3109 static struct opcode group7_rm7[] = {
3111 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
3115 static struct opcode group1[] = {
3126 static struct opcode group1A[] = {
3127 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
3130 static struct opcode group3[] = {
3131 I(DstMem | SrcImm | ModRM, em_test),
3132 I(DstMem | SrcImm | ModRM, em_test),
3133 I(DstMem | SrcNone | ModRM | Lock, em_not),
3134 I(DstMem | SrcNone | ModRM | Lock, em_neg),
3135 I(SrcMem | ModRM, em_mul_ex),
3136 I(SrcMem | ModRM, em_imul_ex),
3137 I(SrcMem | ModRM, em_div_ex),
3138 I(SrcMem | ModRM, em_idiv_ex),
3141 static struct opcode group4[] = {
3142 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
3146 static struct opcode group5[] = {
3147 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3148 D(SrcMem | ModRM | Stack),
3149 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3150 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
3151 D(SrcMem | ModRM | Stack), N,
3154 static struct opcode group6[] = {
3155 DI(ModRM | Prot, sldt),
3156 DI(ModRM | Prot, str),
3157 DI(ModRM | Prot | Priv, lldt),
3158 DI(ModRM | Prot | Priv, ltr),
3162 static struct group_dual group7 = { {
3163 DI(ModRM | Mov | DstMem | Priv, sgdt),
3164 DI(ModRM | Mov | DstMem | Priv, sidt),
3165 II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
3166 II(ModRM | SrcMem | Priv, em_lidt, lidt),
3167 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3168 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
3169 II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3171 I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
3173 N, EXT(0, group7_rm3),
3174 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3175 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3178 static struct opcode group8[] = {
3180 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
3181 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
3184 static struct group_dual group9 = { {
3185 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
3187 N, N, N, N, N, N, N, N,
3190 static struct opcode group11[] = {
3191 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
3194 static struct gprefix pfx_0f_6f_0f_7f = {
3195 N, N, N, I(Sse, em_movdqu),
3198 static struct opcode opcode_table[256] = {
3200 I6ALU(Lock, em_add),
3201 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3202 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3205 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3208 I6ALU(Lock, em_adc),
3209 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3210 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3212 I6ALU(Lock, em_sbb),
3213 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3214 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3216 I6ALU(Lock, em_and), N, N,
3218 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3220 I6ALU(Lock, em_xor), N, N,
3222 I6ALU(0, em_cmp), N, N,
3226 X8(I(SrcReg | Stack, em_push)),
3228 X8(I(DstReg | Stack, em_pop)),
3230 I(ImplicitOps | Stack | No64, em_pusha),
3231 I(ImplicitOps | Stack | No64, em_popa),
3232 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3235 I(SrcImm | Mov | Stack, em_push),
3236 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3237 I(SrcImmByte | Mov | Stack, em_push),
3238 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3239 D2bvIP(DstDI | SrcDX | Mov | String, ins, check_perm_in), /* insb, insw/insd */
3240 D2bvIP(SrcSI | DstDX | String, outs, check_perm_out), /* outsb, outsw/outsd */
3244 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
3245 G(DstMem | SrcImm | ModRM | Group, group1),
3246 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
3247 G(DstMem | SrcImmByte | ModRM | Group, group1),
3248 I2bv(DstMem | SrcReg | ModRM, em_test),
3249 I2bv(DstMem | SrcReg | ModRM | Lock, em_xchg),
3251 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
3252 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3253 I(DstMem | SrcNone | ModRM | Mov, em_mov_rm_sreg),
3254 D(ModRM | SrcMem | NoAccess | DstReg),
3255 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3258 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3260 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3261 I(SrcImmFAddr | No64, em_call_far), N,
3262 II(ImplicitOps | Stack, em_pushf, pushf),
3263 II(ImplicitOps | Stack, em_popf, popf), N, N,
3265 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3266 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
3267 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3268 I2bv(SrcSI | DstDI | String, em_cmp),
3270 I2bv(DstAcc | SrcImm, em_test),
3271 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3272 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3273 I2bv(SrcAcc | DstDI | String, em_cmp),
3275 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3277 X8(I(DstReg | SrcImm | Mov, em_mov)),
3279 D2bv(DstMem | SrcImmByte | ModRM),
3280 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3281 I(ImplicitOps | Stack, em_ret),
3282 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3283 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3284 G(ByteOp, group11), G(0, group11),
3286 N, N, N, I(ImplicitOps | Stack, em_ret_far),
3287 D(ImplicitOps), DI(SrcImmByte, intn),
3288 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3290 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3293 N, N, N, N, N, N, N, N,
3295 X3(I(SrcImmByte, em_loop)),
3296 I(SrcImmByte, em_jcxz),
3297 D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
3298 D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
3300 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3301 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3302 D2bvIP(SrcDX | DstAcc, in, check_perm_in),
3303 D2bvIP(SrcAcc | DstDX, out, check_perm_out),
3305 N, DI(ImplicitOps, icebp), N, N,
3306 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3307 G(ByteOp, group3), G(0, group3),
3309 D(ImplicitOps), D(ImplicitOps),
3310 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3311 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3314 static struct opcode twobyte_table[256] = {
3316 G(0, group6), GD(0, &group7), N, N,
3317 N, I(ImplicitOps | VendorSpecific, em_syscall),
3318 II(ImplicitOps | Priv, em_clts, clts), N,
3319 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3320 N, D(ImplicitOps | ModRM), N, N,
3322 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3324 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3325 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3326 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3327 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3329 N, N, N, N, N, N, N, N,
3331 DI(ImplicitOps | Priv, wrmsr),
3332 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3333 DI(ImplicitOps | Priv, rdmsr),
3334 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3335 I(ImplicitOps | VendorSpecific, em_sysenter),
3336 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3338 N, N, N, N, N, N, N, N,
3340 X16(D(DstReg | SrcMem | ModRM | Mov)),
3342 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3347 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3352 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3356 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3358 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3359 DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3360 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3361 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3363 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3364 DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3365 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3366 D(DstMem | SrcReg | Src2CL | ModRM),
3367 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3369 D2bv(DstMem | SrcReg | ModRM | Lock),
3370 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3371 D(DstMem | SrcReg | ModRM | BitOp | Lock),
3372 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3373 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3374 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3377 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3378 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3379 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3381 D2bv(DstMem | SrcReg | ModRM | Lock),
3382 N, D(DstMem | SrcReg | ModRM | Mov),
3383 N, N, N, GD(0, &group9),
3384 N, N, N, N, N, N, N, N,
3386 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3388 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3390 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3406 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3410 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3416 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3417 unsigned size, bool sign_extension)
3419 int rc = X86EMUL_CONTINUE;
3423 op->addr.mem.ea = ctxt->_eip;
3424 /* NB. Immediates are sign-extended as necessary. */
3425 switch (op->bytes) {
3427 op->val = insn_fetch(s8, ctxt);
3430 op->val = insn_fetch(s16, ctxt);
3433 op->val = insn_fetch(s32, ctxt);
3436 if (!sign_extension) {
3437 switch (op->bytes) {
3445 op->val &= 0xffffffff;
3453 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
3456 int rc = X86EMUL_CONTINUE;
3460 decode_register_operand(ctxt, op,
3462 ctxt->twobyte && (ctxt->b == 0xb6 || ctxt->b == 0xb7));
3465 rc = decode_imm(ctxt, op, 1, false);
3468 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3472 if ((ctxt->d & BitOp) && op == &ctxt->dst)
3473 fetch_bit_operand(ctxt);
3474 op->orig_val = op->val;
3477 ctxt->memop.bytes = 8;
3481 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3482 op->addr.reg = &ctxt->regs[VCPU_REGS_RAX];
3483 fetch_register_operand(op);
3484 op->orig_val = op->val;
3488 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3490 register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]);
3491 op->addr.mem.seg = VCPU_SREG_ES;
3497 op->addr.reg = &ctxt->regs[VCPU_REGS_RDX];
3498 fetch_register_operand(op);
3502 op->val = ctxt->regs[VCPU_REGS_RCX] & 0xff;
3505 rc = decode_imm(ctxt, op, 1, true);
3512 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
3515 ctxt->memop.bytes = 2;
3518 ctxt->memop.bytes = 4;
3521 rc = decode_imm(ctxt, op, 2, false);
3524 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
3528 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3530 register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]);
3531 op->addr.mem.seg = seg_override(ctxt);
3536 op->addr.mem.ea = ctxt->_eip;
3537 op->bytes = ctxt->op_bytes + 2;
3538 insn_fetch_arr(op->valptr, op->bytes, ctxt);
3541 ctxt->memop.bytes = ctxt->op_bytes + 2;
3544 op->val = VCPU_SREG_ES;
3547 op->val = VCPU_SREG_CS;
3550 op->val = VCPU_SREG_SS;
3553 op->val = VCPU_SREG_DS;
3556 op->val = VCPU_SREG_FS;
3559 op->val = VCPU_SREG_GS;
3562 /* Special instructions do their own operand decoding. */
3564 op->type = OP_NONE; /* Disable writeback. */
3572 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3574 int rc = X86EMUL_CONTINUE;
3575 int mode = ctxt->mode;
3576 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3577 bool op_prefix = false;
3578 struct opcode opcode;
3580 ctxt->memop.type = OP_NONE;
3581 ctxt->memopp = NULL;
3582 ctxt->_eip = ctxt->eip;
3583 ctxt->fetch.start = ctxt->_eip;
3584 ctxt->fetch.end = ctxt->fetch.start + insn_len;
3586 memcpy(ctxt->fetch.data, insn, insn_len);
3589 case X86EMUL_MODE_REAL:
3590 case X86EMUL_MODE_VM86:
3591 case X86EMUL_MODE_PROT16:
3592 def_op_bytes = def_ad_bytes = 2;
3594 case X86EMUL_MODE_PROT32:
3595 def_op_bytes = def_ad_bytes = 4;
3597 #ifdef CONFIG_X86_64
3598 case X86EMUL_MODE_PROT64:
3604 return EMULATION_FAILED;
3607 ctxt->op_bytes = def_op_bytes;
3608 ctxt->ad_bytes = def_ad_bytes;
3610 /* Legacy prefixes. */
3612 switch (ctxt->b = insn_fetch(u8, ctxt)) {
3613 case 0x66: /* operand-size override */
3615 /* switch between 2/4 bytes */
3616 ctxt->op_bytes = def_op_bytes ^ 6;
3618 case 0x67: /* address-size override */
3619 if (mode == X86EMUL_MODE_PROT64)
3620 /* switch between 4/8 bytes */
3621 ctxt->ad_bytes = def_ad_bytes ^ 12;
3623 /* switch between 2/4 bytes */
3624 ctxt->ad_bytes = def_ad_bytes ^ 6;
3626 case 0x26: /* ES override */
3627 case 0x2e: /* CS override */
3628 case 0x36: /* SS override */
3629 case 0x3e: /* DS override */
3630 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
3632 case 0x64: /* FS override */
3633 case 0x65: /* GS override */
3634 set_seg_override(ctxt, ctxt->b & 7);
3636 case 0x40 ... 0x4f: /* REX */
3637 if (mode != X86EMUL_MODE_PROT64)
3639 ctxt->rex_prefix = ctxt->b;
3641 case 0xf0: /* LOCK */
3642 ctxt->lock_prefix = 1;
3644 case 0xf2: /* REPNE/REPNZ */
3645 case 0xf3: /* REP/REPE/REPZ */
3646 ctxt->rep_prefix = ctxt->b;
3652 /* Any legacy prefix after a REX prefix nullifies its effect. */
3654 ctxt->rex_prefix = 0;
3660 if (ctxt->rex_prefix & 8)
3661 ctxt->op_bytes = 8; /* REX.W */
3663 /* Opcode byte(s). */
3664 opcode = opcode_table[ctxt->b];
3665 /* Two-byte opcode? */
3666 if (ctxt->b == 0x0f) {
3668 ctxt->b = insn_fetch(u8, ctxt);
3669 opcode = twobyte_table[ctxt->b];
3671 ctxt->d = opcode.flags;
3673 while (ctxt->d & GroupMask) {
3674 switch (ctxt->d & GroupMask) {
3676 ctxt->modrm = insn_fetch(u8, ctxt);
3678 goffset = (ctxt->modrm >> 3) & 7;
3679 opcode = opcode.u.group[goffset];
3682 ctxt->modrm = insn_fetch(u8, ctxt);
3684 goffset = (ctxt->modrm >> 3) & 7;
3685 if ((ctxt->modrm >> 6) == 3)
3686 opcode = opcode.u.gdual->mod3[goffset];
3688 opcode = opcode.u.gdual->mod012[goffset];
3691 goffset = ctxt->modrm & 7;
3692 opcode = opcode.u.group[goffset];
3695 if (ctxt->rep_prefix && op_prefix)
3696 return EMULATION_FAILED;
3697 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
3698 switch (simd_prefix) {
3699 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3700 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3701 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3702 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3706 return EMULATION_FAILED;
3709 ctxt->d &= ~(u64)GroupMask;
3710 ctxt->d |= opcode.flags;
3713 ctxt->execute = opcode.u.execute;
3714 ctxt->check_perm = opcode.check_perm;
3715 ctxt->intercept = opcode.intercept;
3718 if (ctxt->d == 0 || (ctxt->d & Undefined))
3719 return EMULATION_FAILED;
3721 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3722 return EMULATION_FAILED;
3724 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
3727 if (ctxt->d & Op3264) {
3728 if (mode == X86EMUL_MODE_PROT64)
3735 ctxt->op_bytes = 16;
3737 /* ModRM and SIB bytes. */
3738 if (ctxt->d & ModRM) {
3739 rc = decode_modrm(ctxt, &ctxt->memop);
3740 if (!ctxt->has_seg_override)
3741 set_seg_override(ctxt, ctxt->modrm_seg);
3742 } else if (ctxt->d & MemAbs)
3743 rc = decode_abs(ctxt, &ctxt->memop);
3744 if (rc != X86EMUL_CONTINUE)
3747 if (!ctxt->has_seg_override)
3748 set_seg_override(ctxt, VCPU_SREG_DS);
3750 ctxt->memop.addr.mem.seg = seg_override(ctxt);
3752 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
3753 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
3756 * Decode and fetch the source operand: register, memory
3759 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
3760 if (rc != X86EMUL_CONTINUE)
3764 * Decode and fetch the second source operand: register, memory
3767 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
3768 if (rc != X86EMUL_CONTINUE)
3771 /* Decode and fetch the destination operand: register or memory. */
3772 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
3775 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
3776 ctxt->memopp->addr.mem.ea += ctxt->_eip;
3778 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
3781 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3783 /* The second termination condition only applies for REPE
3784 * and REPNE. Test if the repeat string operation prefix is
3785 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3786 * corresponding termination condition according to:
3787 * - if REPE/REPZ and ZF = 0 then done
3788 * - if REPNE/REPNZ and ZF = 1 then done
3790 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
3791 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
3792 && (((ctxt->rep_prefix == REPE_PREFIX) &&
3793 ((ctxt->eflags & EFLG_ZF) == 0))
3794 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
3795 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3801 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3803 struct x86_emulate_ops *ops = ctxt->ops;
3805 int rc = X86EMUL_CONTINUE;
3806 int saved_dst_type = ctxt->dst.type;
3808 ctxt->mem_read.pos = 0;
3810 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
3811 rc = emulate_ud(ctxt);
3815 /* LOCK prefix is allowed only with some instructions */
3816 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
3817 rc = emulate_ud(ctxt);
3821 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
3822 rc = emulate_ud(ctxt);
3827 && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
3828 || !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
3829 rc = emulate_ud(ctxt);
3833 if ((ctxt->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
3834 rc = emulate_nm(ctxt);
3838 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3839 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3840 X86_ICPT_PRE_EXCEPT);
3841 if (rc != X86EMUL_CONTINUE)
3845 /* Privileged instruction can be executed only in CPL=0 */
3846 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
3847 rc = emulate_gp(ctxt, 0);
3851 /* Instruction can only be executed in protected mode */
3852 if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3853 rc = emulate_ud(ctxt);
3857 /* Do instruction specific permission checks */
3858 if (ctxt->check_perm) {
3859 rc = ctxt->check_perm(ctxt);
3860 if (rc != X86EMUL_CONTINUE)
3864 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3865 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3866 X86_ICPT_POST_EXCEPT);
3867 if (rc != X86EMUL_CONTINUE)
3871 if (ctxt->rep_prefix && (ctxt->d & String)) {
3872 /* All REP prefixes have the same first termination condition */
3873 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) {
3874 ctxt->eip = ctxt->_eip;
3879 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
3880 rc = segmented_read(ctxt, ctxt->src.addr.mem,
3881 ctxt->src.valptr, ctxt->src.bytes);
3882 if (rc != X86EMUL_CONTINUE)
3884 ctxt->src.orig_val64 = ctxt->src.val64;
3887 if (ctxt->src2.type == OP_MEM) {
3888 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
3889 &ctxt->src2.val, ctxt->src2.bytes);
3890 if (rc != X86EMUL_CONTINUE)
3894 if ((ctxt->d & DstMask) == ImplicitOps)
3898 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
3899 /* optimisation - avoid slow emulated read if Mov */
3900 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
3901 &ctxt->dst.val, ctxt->dst.bytes);
3902 if (rc != X86EMUL_CONTINUE)
3905 ctxt->dst.orig_val = ctxt->dst.val;
3909 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3910 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3911 X86_ICPT_POST_MEMACCESS);
3912 if (rc != X86EMUL_CONTINUE)
3916 if (ctxt->execute) {
3917 rc = ctxt->execute(ctxt);
3918 if (rc != X86EMUL_CONTINUE)
3927 case 0x40 ... 0x47: /* inc r16/r32 */
3928 emulate_1op(ctxt, "inc");
3930 case 0x48 ... 0x4f: /* dec r16/r32 */
3931 emulate_1op(ctxt, "dec");
3933 case 0x63: /* movsxd */
3934 if (ctxt->mode != X86EMUL_MODE_PROT64)
3935 goto cannot_emulate;
3936 ctxt->dst.val = (s32) ctxt->src.val;
3938 case 0x6c: /* insb */
3939 case 0x6d: /* insw/insd */
3940 ctxt->src.val = ctxt->regs[VCPU_REGS_RDX];
3942 case 0x6e: /* outsb */
3943 case 0x6f: /* outsw/outsd */
3944 ctxt->dst.val = ctxt->regs[VCPU_REGS_RDX];
3947 case 0x70 ... 0x7f: /* jcc (short) */
3948 if (test_cc(ctxt->b, ctxt->eflags))
3949 jmp_rel(ctxt, ctxt->src.val);
3951 case 0x8d: /* lea r16/r32, m */
3952 ctxt->dst.val = ctxt->src.addr.mem.ea;
3954 case 0x8f: /* pop (sole member of Grp1a) */
3955 rc = em_grp1a(ctxt);
3957 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3958 if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX])
3962 case 0x98: /* cbw/cwde/cdqe */
3963 switch (ctxt->op_bytes) {
3964 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
3965 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
3966 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
3972 case 0xcc: /* int3 */
3973 rc = emulate_int(ctxt, 3);
3975 case 0xcd: /* int n */
3976 rc = emulate_int(ctxt, ctxt->src.val);
3978 case 0xce: /* into */
3979 if (ctxt->eflags & EFLG_OF)
3980 rc = emulate_int(ctxt, 4);
3982 case 0xd0 ... 0xd1: /* Grp2 */
3985 case 0xd2 ... 0xd3: /* Grp2 */
3986 ctxt->src.val = ctxt->regs[VCPU_REGS_RCX];
3989 case 0xe4: /* inb */
3992 case 0xe6: /* outb */
3993 case 0xe7: /* out */
3995 case 0xe9: /* jmp rel */
3996 case 0xeb: /* jmp rel short */
3997 jmp_rel(ctxt, ctxt->src.val);
3998 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4000 case 0xec: /* in al,dx */
4001 case 0xed: /* in (e/r)ax,dx */
4003 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
4005 goto done; /* IO is needed */
4007 case 0xee: /* out dx,al */
4008 case 0xef: /* out dx,(e/r)ax */
4010 ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
4012 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4014 case 0xf4: /* hlt */
4015 ctxt->ops->halt(ctxt);
4017 case 0xf5: /* cmc */
4018 /* complement carry flag from eflags reg */
4019 ctxt->eflags ^= EFLG_CF;
4021 case 0xf8: /* clc */
4022 ctxt->eflags &= ~EFLG_CF;
4024 case 0xf9: /* stc */
4025 ctxt->eflags |= EFLG_CF;
4027 case 0xfc: /* cld */
4028 ctxt->eflags &= ~EFLG_DF;
4030 case 0xfd: /* std */
4031 ctxt->eflags |= EFLG_DF;
4033 case 0xfe: /* Grp4 */
4034 rc = em_grp45(ctxt);
4036 case 0xff: /* Grp5 */
4037 rc = em_grp45(ctxt);
4040 goto cannot_emulate;
4043 if (rc != X86EMUL_CONTINUE)
4047 rc = writeback(ctxt);
4048 if (rc != X86EMUL_CONTINUE)
4052 * restore dst type in case the decoding will be reused
4053 * (happens for string instruction )
4055 ctxt->dst.type = saved_dst_type;
4057 if ((ctxt->d & SrcMask) == SrcSI)
4058 string_addr_inc(ctxt, seg_override(ctxt),
4059 VCPU_REGS_RSI, &ctxt->src);
4061 if ((ctxt->d & DstMask) == DstDI)
4062 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4065 if (ctxt->rep_prefix && (ctxt->d & String)) {
4066 struct read_cache *r = &ctxt->io_read;
4067 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
4069 if (!string_insn_completed(ctxt)) {
4071 * Re-enter guest when pio read ahead buffer is empty
4072 * or, if it is not used, after each 1024 iteration.
4074 if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) &&
4075 (r->end == 0 || r->end != r->pos)) {
4077 * Reset read cache. Usually happens before
4078 * decode, but since instruction is restarted
4079 * we have to do it here.
4081 ctxt->mem_read.end = 0;
4082 return EMULATION_RESTART;
4084 goto done; /* skip rip writeback */
4088 ctxt->eip = ctxt->_eip;
4091 if (rc == X86EMUL_PROPAGATE_FAULT)
4092 ctxt->have_exception = true;
4093 if (rc == X86EMUL_INTERCEPTED)
4094 return EMULATION_INTERCEPTED;
4096 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4100 case 0x09: /* wbinvd */
4101 (ctxt->ops->wbinvd)(ctxt);
4103 case 0x08: /* invd */
4104 case 0x0d: /* GrpP (prefetch) */
4105 case 0x18: /* Grp16 (prefetch/nop) */
4107 case 0x20: /* mov cr, reg */
4108 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4110 case 0x21: /* mov from dr to reg */
4111 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4113 case 0x22: /* mov reg, cr */
4114 if (ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val)) {
4115 emulate_gp(ctxt, 0);
4116 rc = X86EMUL_PROPAGATE_FAULT;
4119 ctxt->dst.type = OP_NONE;
4121 case 0x23: /* mov from reg to dr */
4122 if (ops->set_dr(ctxt, ctxt->modrm_reg, ctxt->src.val &
4123 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4124 ~0ULL : ~0U)) < 0) {
4125 /* #UD condition is already handled by the code above */
4126 emulate_gp(ctxt, 0);
4127 rc = X86EMUL_PROPAGATE_FAULT;
4131 ctxt->dst.type = OP_NONE; /* no writeback */
4135 msr_data = (u32)ctxt->regs[VCPU_REGS_RAX]
4136 | ((u64)ctxt->regs[VCPU_REGS_RDX] << 32);
4137 if (ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data)) {
4138 emulate_gp(ctxt, 0);
4139 rc = X86EMUL_PROPAGATE_FAULT;
4142 rc = X86EMUL_CONTINUE;
4146 if (ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data)) {
4147 emulate_gp(ctxt, 0);
4148 rc = X86EMUL_PROPAGATE_FAULT;
4151 ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data;
4152 ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32;
4154 rc = X86EMUL_CONTINUE;
4156 case 0x40 ... 0x4f: /* cmov */
4157 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4158 if (!test_cc(ctxt->b, ctxt->eflags))
4159 ctxt->dst.type = OP_NONE; /* no writeback */
4161 case 0x80 ... 0x8f: /* jnz rel, etc*/
4162 if (test_cc(ctxt->b, ctxt->eflags))
4163 jmp_rel(ctxt, ctxt->src.val);
4165 case 0x90 ... 0x9f: /* setcc r/m8 */
4166 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4170 ctxt->dst.type = OP_NONE;
4171 /* only subword offset */
4172 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
4173 emulate_2op_SrcV_nobyte(ctxt, "bt");
4175 case 0xa4: /* shld imm8, r, r/m */
4176 case 0xa5: /* shld cl, r, r/m */
4177 emulate_2op_cl(ctxt, "shld");
4181 emulate_2op_SrcV_nobyte(ctxt, "bts");
4183 case 0xac: /* shrd imm8, r, r/m */
4184 case 0xad: /* shrd cl, r, r/m */
4185 emulate_2op_cl(ctxt, "shrd");
4187 case 0xae: /* clflush */
4189 case 0xb0 ... 0xb1: /* cmpxchg */
4191 * Save real source value, then compare EAX against
4194 ctxt->src.orig_val = ctxt->src.val;
4195 ctxt->src.val = ctxt->regs[VCPU_REGS_RAX];
4196 emulate_2op_SrcV(ctxt, "cmp");
4197 if (ctxt->eflags & EFLG_ZF) {
4198 /* Success: write back to memory. */
4199 ctxt->dst.val = ctxt->src.orig_val;
4201 /* Failure: write the value we saw to EAX. */
4202 ctxt->dst.type = OP_REG;
4203 ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX];
4208 emulate_2op_SrcV_nobyte(ctxt, "btr");
4210 case 0xb6 ... 0xb7: /* movzx */
4211 ctxt->dst.bytes = ctxt->op_bytes;
4212 ctxt->dst.val = (ctxt->d & ByteOp) ? (u8) ctxt->src.val
4213 : (u16) ctxt->src.val;
4215 case 0xba: /* Grp8 */
4216 switch (ctxt->modrm_reg & 3) {
4229 emulate_2op_SrcV_nobyte(ctxt, "btc");
4231 case 0xbc: { /* bsf */
4233 __asm__ ("bsf %2, %0; setz %1"
4234 : "=r"(ctxt->dst.val), "=q"(zf)
4235 : "r"(ctxt->src.val));
4236 ctxt->eflags &= ~X86_EFLAGS_ZF;
4238 ctxt->eflags |= X86_EFLAGS_ZF;
4239 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4243 case 0xbd: { /* bsr */
4245 __asm__ ("bsr %2, %0; setz %1"
4246 : "=r"(ctxt->dst.val), "=q"(zf)
4247 : "r"(ctxt->src.val));
4248 ctxt->eflags &= ~X86_EFLAGS_ZF;
4250 ctxt->eflags |= X86_EFLAGS_ZF;
4251 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4255 case 0xbe ... 0xbf: /* movsx */
4256 ctxt->dst.bytes = ctxt->op_bytes;
4257 ctxt->dst.val = (ctxt->d & ByteOp) ? (s8) ctxt->src.val :
4258 (s16) ctxt->src.val;
4260 case 0xc0 ... 0xc1: /* xadd */
4261 emulate_2op_SrcV(ctxt, "add");
4262 /* Write back the register source. */
4263 ctxt->src.val = ctxt->dst.orig_val;
4264 write_register_operand(&ctxt->src);
4266 case 0xc3: /* movnti */
4267 ctxt->dst.bytes = ctxt->op_bytes;
4268 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4269 (u64) ctxt->src.val;
4271 case 0xc7: /* Grp9 (cmpxchg8b) */
4275 goto cannot_emulate;
4278 if (rc != X86EMUL_CONTINUE)
4284 return EMULATION_FAILED;