1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
35 #define OpImplicit 1ull /* No generic decode */
36 #define OpReg 2ull /* Register */
37 #define OpMem 3ull /* Memory */
38 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
39 #define OpDI 5ull /* ES:DI/EDI/RDI */
40 #define OpMem64 6ull /* Memory, 64-bit */
41 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
42 #define OpDX 8ull /* DX register */
43 #define OpCL 9ull /* CL register (for shifts) */
44 #define OpImmByte 10ull /* 8-bit sign extended immediate */
45 #define OpOne 11ull /* Implied 1 */
46 #define OpImm 12ull /* Sign extended immediate */
47 #define OpMem16 13ull /* Memory operand (16-bit). */
48 #define OpMem32 14ull /* Memory operand (32-bit). */
49 #define OpImmU 15ull /* Immediate operand, zero extended */
50 #define OpSI 16ull /* SI/ESI/RSI */
51 #define OpImmFAddr 17ull /* Immediate far address */
52 #define OpMemFAddr 18ull /* Far address in memory */
53 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
54 #define OpES 20ull /* ES */
55 #define OpCS 21ull /* CS */
56 #define OpSS 22ull /* SS */
57 #define OpDS 23ull /* DS */
58 #define OpFS 24ull /* FS */
59 #define OpGS 25ull /* GS */
60 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpBits 5 /* Width of operand field */
63 #define OpMask ((1ull << OpBits) - 1)
66 * Opcode effective-address decode tables.
67 * Note that we only emulate instructions that have at least one memory
68 * operand (excluding implicit stack references). We assume that stack
69 * references and instruction fetches will never occur in special memory
70 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
74 /* Operand sizes: 8-bit operands or specified/overridden size. */
75 #define ByteOp (1<<0) /* 8-bit operands. */
76 /* Destination operand type. */
78 #define ImplicitOps (OpImplicit << DstShift)
79 #define DstReg (OpReg << DstShift)
80 #define DstMem (OpMem << DstShift)
81 #define DstAcc (OpAcc << DstShift)
82 #define DstDI (OpDI << DstShift)
83 #define DstMem64 (OpMem64 << DstShift)
84 #define DstImmUByte (OpImmUByte << DstShift)
85 #define DstDX (OpDX << DstShift)
86 #define DstMask (OpMask << DstShift)
87 /* Source operand type. */
89 #define SrcNone (OpNone << SrcShift)
90 #define SrcReg (OpReg << SrcShift)
91 #define SrcMem (OpMem << SrcShift)
92 #define SrcMem16 (OpMem16 << SrcShift)
93 #define SrcMem32 (OpMem32 << SrcShift)
94 #define SrcImm (OpImm << SrcShift)
95 #define SrcImmByte (OpImmByte << SrcShift)
96 #define SrcOne (OpOne << SrcShift)
97 #define SrcImmUByte (OpImmUByte << SrcShift)
98 #define SrcImmU (OpImmU << SrcShift)
99 #define SrcSI (OpSI << SrcShift)
100 #define SrcImmFAddr (OpImmFAddr << SrcShift)
101 #define SrcMemFAddr (OpMemFAddr << SrcShift)
102 #define SrcAcc (OpAcc << SrcShift)
103 #define SrcImmU16 (OpImmU16 << SrcShift)
104 #define SrcDX (OpDX << SrcShift)
105 #define SrcMem8 (OpMem8 << SrcShift)
106 #define SrcMask (OpMask << SrcShift)
107 #define BitOp (1<<11)
108 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
109 #define String (1<<13) /* String instruction (rep capable) */
110 #define Stack (1<<14) /* Stack instruction (push/pop) */
111 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
112 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
113 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
114 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
115 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
116 #define Sse (1<<18) /* SSE Vector instruction */
117 /* Generic ModRM decode. */
118 #define ModRM (1<<19)
119 /* Destination is only written; never read. */
122 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
123 #define VendorSpecific (1<<22) /* Vendor specific instruction */
124 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
125 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
126 #define Undefined (1<<25) /* No Such Instruction */
127 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
128 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
130 #define PageTable (1 << 29) /* instruction used to write page table */
131 /* Source 2 operand type */
132 #define Src2Shift (30)
133 #define Src2None (OpNone << Src2Shift)
134 #define Src2CL (OpCL << Src2Shift)
135 #define Src2ImmByte (OpImmByte << Src2Shift)
136 #define Src2One (OpOne << Src2Shift)
137 #define Src2Imm (OpImm << Src2Shift)
138 #define Src2ES (OpES << Src2Shift)
139 #define Src2CS (OpCS << Src2Shift)
140 #define Src2SS (OpSS << Src2Shift)
141 #define Src2DS (OpDS << Src2Shift)
142 #define Src2FS (OpFS << Src2Shift)
143 #define Src2GS (OpGS << Src2Shift)
144 #define Src2Mask (OpMask << Src2Shift)
146 #define X2(x...) x, x
147 #define X3(x...) X2(x), x
148 #define X4(x...) X2(x), X2(x)
149 #define X5(x...) X4(x), x
150 #define X6(x...) X4(x), X2(x)
151 #define X7(x...) X4(x), X3(x)
152 #define X8(x...) X4(x), X4(x)
153 #define X16(x...) X8(x), X8(x)
159 int (*execute)(struct x86_emulate_ctxt *ctxt);
160 struct opcode *group;
161 struct group_dual *gdual;
162 struct gprefix *gprefix;
164 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
168 struct opcode mod012[8];
169 struct opcode mod3[8];
173 struct opcode pfx_no;
174 struct opcode pfx_66;
175 struct opcode pfx_f2;
176 struct opcode pfx_f3;
179 /* EFLAGS bit definitions. */
180 #define EFLG_ID (1<<21)
181 #define EFLG_VIP (1<<20)
182 #define EFLG_VIF (1<<19)
183 #define EFLG_AC (1<<18)
184 #define EFLG_VM (1<<17)
185 #define EFLG_RF (1<<16)
186 #define EFLG_IOPL (3<<12)
187 #define EFLG_NT (1<<14)
188 #define EFLG_OF (1<<11)
189 #define EFLG_DF (1<<10)
190 #define EFLG_IF (1<<9)
191 #define EFLG_TF (1<<8)
192 #define EFLG_SF (1<<7)
193 #define EFLG_ZF (1<<6)
194 #define EFLG_AF (1<<4)
195 #define EFLG_PF (1<<2)
196 #define EFLG_CF (1<<0)
198 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
199 #define EFLG_RESERVED_ONE_MASK 2
202 * Instruction emulation:
203 * Most instructions are emulated directly via a fragment of inline assembly
204 * code. This allows us to save/restore EFLAGS and thus very easily pick up
205 * any modified flags.
208 #if defined(CONFIG_X86_64)
209 #define _LO32 "k" /* force 32-bit operand */
210 #define _STK "%%rsp" /* stack pointer */
211 #elif defined(__i386__)
212 #define _LO32 "" /* force 32-bit operand */
213 #define _STK "%%esp" /* stack pointer */
217 * These EFLAGS bits are restored from saved value during emulation, and
218 * any changes are written back to the saved value after emulation.
220 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
222 /* Before executing instruction: restore necessary bits in EFLAGS. */
223 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
224 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
225 "movl %"_sav",%"_LO32 _tmp"; " \
228 "movl %"_msk",%"_LO32 _tmp"; " \
229 "andl %"_LO32 _tmp",("_STK"); " \
231 "notl %"_LO32 _tmp"; " \
232 "andl %"_LO32 _tmp",("_STK"); " \
233 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
235 "orl %"_LO32 _tmp",("_STK"); " \
239 /* After executing instruction: write-back necessary bits in EFLAGS. */
240 #define _POST_EFLAGS(_sav, _msk, _tmp) \
241 /* _sav |= EFLAGS & _msk; */ \
244 "andl %"_msk",%"_LO32 _tmp"; " \
245 "orl %"_LO32 _tmp",%"_sav"; "
253 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
255 __asm__ __volatile__ ( \
256 _PRE_EFLAGS("0", "4", "2") \
257 _op _suffix " %"_x"3,%1; " \
258 _POST_EFLAGS("0", "4", "2") \
259 : "=m" ((ctxt)->eflags), \
260 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
262 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
266 /* Raw emulation: instruction has two explicit operands. */
267 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
269 unsigned long _tmp; \
271 switch ((ctxt)->dst.bytes) { \
273 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
276 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
279 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
284 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
286 unsigned long _tmp; \
287 switch ((ctxt)->dst.bytes) { \
289 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
292 __emulate_2op_nobyte(ctxt, _op, \
293 _wx, _wy, _lx, _ly, _qx, _qy); \
298 /* Source operand is byte-sized and may be restricted to just %cl. */
299 #define emulate_2op_SrcB(ctxt, _op) \
300 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
302 /* Source operand is byte, word, long or quad sized. */
303 #define emulate_2op_SrcV(ctxt, _op) \
304 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
306 /* Source operand is word, long or quad sized. */
307 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
308 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
310 /* Instruction has three operands and one operand is stored in ECX register */
311 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
313 unsigned long _tmp; \
314 _type _clv = (ctxt)->src2.val; \
315 _type _srcv = (ctxt)->src.val; \
316 _type _dstv = (ctxt)->dst.val; \
318 __asm__ __volatile__ ( \
319 _PRE_EFLAGS("0", "5", "2") \
320 _op _suffix " %4,%1 \n" \
321 _POST_EFLAGS("0", "5", "2") \
322 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
323 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
326 (ctxt)->src2.val = (unsigned long) _clv; \
327 (ctxt)->src2.val = (unsigned long) _srcv; \
328 (ctxt)->dst.val = (unsigned long) _dstv; \
331 #define emulate_2op_cl(ctxt, _op) \
333 switch ((ctxt)->dst.bytes) { \
335 __emulate_2op_cl(ctxt, _op, "w", u16); \
338 __emulate_2op_cl(ctxt, _op, "l", u32); \
341 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
346 #define __emulate_1op(ctxt, _op, _suffix) \
348 unsigned long _tmp; \
350 __asm__ __volatile__ ( \
351 _PRE_EFLAGS("0", "3", "2") \
352 _op _suffix " %1; " \
353 _POST_EFLAGS("0", "3", "2") \
354 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
356 : "i" (EFLAGS_MASK)); \
359 /* Instruction has only one explicit operand (no source operand). */
360 #define emulate_1op(ctxt, _op) \
362 switch ((ctxt)->dst.bytes) { \
363 case 1: __emulate_1op(ctxt, _op, "b"); break; \
364 case 2: __emulate_1op(ctxt, _op, "w"); break; \
365 case 4: __emulate_1op(ctxt, _op, "l"); break; \
366 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
370 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
372 unsigned long _tmp; \
373 ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX]; \
374 ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX]; \
376 __asm__ __volatile__ ( \
377 _PRE_EFLAGS("0", "5", "1") \
379 _op _suffix " %6; " \
381 _POST_EFLAGS("0", "5", "1") \
382 ".pushsection .fixup,\"ax\" \n\t" \
383 "3: movb $1, %4 \n\t" \
386 _ASM_EXTABLE(1b, 3b) \
387 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
388 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
389 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val), \
390 "a" (*rax), "d" (*rdx)); \
393 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
394 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
396 switch((ctxt)->src.bytes) { \
398 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
401 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
404 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
407 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
412 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
413 enum x86_intercept intercept,
414 enum x86_intercept_stage stage)
416 struct x86_instruction_info info = {
417 .intercept = intercept,
418 .rep_prefix = ctxt->rep_prefix,
419 .modrm_mod = ctxt->modrm_mod,
420 .modrm_reg = ctxt->modrm_reg,
421 .modrm_rm = ctxt->modrm_rm,
422 .src_val = ctxt->src.val64,
423 .src_bytes = ctxt->src.bytes,
424 .dst_bytes = ctxt->dst.bytes,
425 .ad_bytes = ctxt->ad_bytes,
426 .next_rip = ctxt->eip,
429 return ctxt->ops->intercept(ctxt, &info, stage);
432 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
434 return (1UL << (ctxt->ad_bytes << 3)) - 1;
437 /* Access/update address held in a register, based on addressing mode. */
438 static inline unsigned long
439 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
441 if (ctxt->ad_bytes == sizeof(unsigned long))
444 return reg & ad_mask(ctxt);
447 static inline unsigned long
448 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
450 return address_mask(ctxt, reg);
454 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
456 if (ctxt->ad_bytes == sizeof(unsigned long))
459 *reg = (*reg & ~ad_mask(ctxt)) | ((*reg + inc) & ad_mask(ctxt));
462 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
464 register_address_increment(ctxt, &ctxt->_eip, rel);
467 static u32 desc_limit_scaled(struct desc_struct *desc)
469 u32 limit = get_desc_limit(desc);
471 return desc->g ? (limit << 12) | 0xfff : limit;
474 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
476 ctxt->has_seg_override = true;
477 ctxt->seg_override = seg;
480 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
482 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
485 return ctxt->ops->get_cached_segment_base(ctxt, seg);
488 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
490 if (!ctxt->has_seg_override)
493 return ctxt->seg_override;
496 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
497 u32 error, bool valid)
499 ctxt->exception.vector = vec;
500 ctxt->exception.error_code = error;
501 ctxt->exception.error_code_valid = valid;
502 return X86EMUL_PROPAGATE_FAULT;
505 static int emulate_db(struct x86_emulate_ctxt *ctxt)
507 return emulate_exception(ctxt, DB_VECTOR, 0, false);
510 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
512 return emulate_exception(ctxt, GP_VECTOR, err, true);
515 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
517 return emulate_exception(ctxt, SS_VECTOR, err, true);
520 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
522 return emulate_exception(ctxt, UD_VECTOR, 0, false);
525 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
527 return emulate_exception(ctxt, TS_VECTOR, err, true);
530 static int emulate_de(struct x86_emulate_ctxt *ctxt)
532 return emulate_exception(ctxt, DE_VECTOR, 0, false);
535 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
537 return emulate_exception(ctxt, NM_VECTOR, 0, false);
540 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
543 struct desc_struct desc;
545 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
549 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
554 struct desc_struct desc;
556 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
557 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
560 static int __linearize(struct x86_emulate_ctxt *ctxt,
561 struct segmented_address addr,
562 unsigned size, bool write, bool fetch,
565 struct desc_struct desc;
572 la = seg_base(ctxt, addr.seg) + addr.ea;
573 switch (ctxt->mode) {
574 case X86EMUL_MODE_REAL:
576 case X86EMUL_MODE_PROT64:
577 if (((signed long)la << 16) >> 16 != la)
578 return emulate_gp(ctxt, 0);
581 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
585 /* code segment or read-only data segment */
586 if (((desc.type & 8) || !(desc.type & 2)) && write)
588 /* unreadable code segment */
589 if (!fetch && (desc.type & 8) && !(desc.type & 2))
591 lim = desc_limit_scaled(&desc);
592 if ((desc.type & 8) || !(desc.type & 4)) {
593 /* expand-up segment */
594 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
597 /* exapand-down segment */
598 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
600 lim = desc.d ? 0xffffffff : 0xffff;
601 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
604 cpl = ctxt->ops->cpl(ctxt);
607 if (!(desc.type & 8)) {
611 } else if ((desc.type & 8) && !(desc.type & 4)) {
612 /* nonconforming code segment */
615 } else if ((desc.type & 8) && (desc.type & 4)) {
616 /* conforming code segment */
622 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
625 return X86EMUL_CONTINUE;
627 if (addr.seg == VCPU_SREG_SS)
628 return emulate_ss(ctxt, addr.seg);
630 return emulate_gp(ctxt, addr.seg);
633 static int linearize(struct x86_emulate_ctxt *ctxt,
634 struct segmented_address addr,
635 unsigned size, bool write,
638 return __linearize(ctxt, addr, size, write, false, linear);
642 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
643 struct segmented_address addr,
650 rc = linearize(ctxt, addr, size, false, &linear);
651 if (rc != X86EMUL_CONTINUE)
653 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
657 * Fetch the next byte of the instruction being emulated which is pointed to
658 * by ctxt->_eip, then increment ctxt->_eip.
660 * Also prefetch the remaining bytes of the instruction without crossing page
661 * boundary if they are not in fetch_cache yet.
663 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
665 struct fetch_cache *fc = &ctxt->fetch;
669 if (ctxt->_eip == fc->end) {
670 unsigned long linear;
671 struct segmented_address addr = { .seg = VCPU_SREG_CS,
673 cur_size = fc->end - fc->start;
674 size = min(15UL - cur_size,
675 PAGE_SIZE - offset_in_page(ctxt->_eip));
676 rc = __linearize(ctxt, addr, size, false, true, &linear);
677 if (unlikely(rc != X86EMUL_CONTINUE))
679 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
680 size, &ctxt->exception);
681 if (unlikely(rc != X86EMUL_CONTINUE))
685 *dest = fc->data[ctxt->_eip - fc->start];
687 return X86EMUL_CONTINUE;
690 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
691 void *dest, unsigned size)
695 /* x86 instructions are limited to 15 bytes. */
696 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
697 return X86EMUL_UNHANDLEABLE;
699 rc = do_insn_fetch_byte(ctxt, dest++);
700 if (rc != X86EMUL_CONTINUE)
703 return X86EMUL_CONTINUE;
706 /* Fetch next part of the instruction being emulated. */
707 #define insn_fetch(_type, _ctxt) \
708 ({ unsigned long _x; \
709 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
710 if (rc != X86EMUL_CONTINUE) \
715 #define insn_fetch_arr(_arr, _size, _ctxt) \
716 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
717 if (rc != X86EMUL_CONTINUE) \
722 * Given the 'reg' portion of a ModRM byte, and a register block, return a
723 * pointer into the block that addresses the relevant register.
724 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
726 static void *decode_register(u8 modrm_reg, unsigned long *regs,
731 p = ®s[modrm_reg];
732 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
733 p = (unsigned char *)®s[modrm_reg & 3] + 1;
737 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
738 struct segmented_address addr,
739 u16 *size, unsigned long *address, int op_bytes)
746 rc = segmented_read_std(ctxt, addr, size, 2);
747 if (rc != X86EMUL_CONTINUE)
750 rc = segmented_read_std(ctxt, addr, address, op_bytes);
754 static int test_cc(unsigned int condition, unsigned int flags)
758 switch ((condition & 15) >> 1) {
760 rc |= (flags & EFLG_OF);
762 case 1: /* b/c/nae */
763 rc |= (flags & EFLG_CF);
766 rc |= (flags & EFLG_ZF);
769 rc |= (flags & (EFLG_CF|EFLG_ZF));
772 rc |= (flags & EFLG_SF);
775 rc |= (flags & EFLG_PF);
778 rc |= (flags & EFLG_ZF);
781 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
785 /* Odd condition identifiers (lsb == 1) have inverted sense. */
786 return (!!rc ^ (condition & 1));
789 static void fetch_register_operand(struct operand *op)
793 op->val = *(u8 *)op->addr.reg;
796 op->val = *(u16 *)op->addr.reg;
799 op->val = *(u32 *)op->addr.reg;
802 op->val = *(u64 *)op->addr.reg;
807 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
809 ctxt->ops->get_fpu(ctxt);
811 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
812 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
813 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
814 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
815 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
816 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
817 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
818 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
820 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
821 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
822 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
823 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
824 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
825 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
826 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
827 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
831 ctxt->ops->put_fpu(ctxt);
834 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
837 ctxt->ops->get_fpu(ctxt);
839 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
840 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
841 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
842 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
843 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
844 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
845 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
846 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
848 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
849 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
850 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
851 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
852 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
853 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
854 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
855 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
859 ctxt->ops->put_fpu(ctxt);
862 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
865 unsigned reg = ctxt->modrm_reg;
866 int highbyte_regs = ctxt->rex_prefix == 0;
868 if (!(ctxt->d & ModRM))
869 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
875 read_sse_reg(ctxt, &op->vec_val, reg);
880 if (ctxt->d & ByteOp) {
881 op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
884 op->addr.reg = decode_register(reg, ctxt->regs, 0);
885 op->bytes = ctxt->op_bytes;
887 fetch_register_operand(op);
888 op->orig_val = op->val;
891 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
895 int index_reg = 0, base_reg = 0, scale;
896 int rc = X86EMUL_CONTINUE;
899 if (ctxt->rex_prefix) {
900 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
901 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
902 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
905 ctxt->modrm = insn_fetch(u8, ctxt);
906 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
907 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
908 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
909 ctxt->modrm_seg = VCPU_SREG_DS;
911 if (ctxt->modrm_mod == 3) {
913 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
914 op->addr.reg = decode_register(ctxt->modrm_rm,
915 ctxt->regs, ctxt->d & ByteOp);
919 op->addr.xmm = ctxt->modrm_rm;
920 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
923 fetch_register_operand(op);
929 if (ctxt->ad_bytes == 2) {
930 unsigned bx = ctxt->regs[VCPU_REGS_RBX];
931 unsigned bp = ctxt->regs[VCPU_REGS_RBP];
932 unsigned si = ctxt->regs[VCPU_REGS_RSI];
933 unsigned di = ctxt->regs[VCPU_REGS_RDI];
935 /* 16-bit ModR/M decode. */
936 switch (ctxt->modrm_mod) {
938 if (ctxt->modrm_rm == 6)
939 modrm_ea += insn_fetch(u16, ctxt);
942 modrm_ea += insn_fetch(s8, ctxt);
945 modrm_ea += insn_fetch(u16, ctxt);
948 switch (ctxt->modrm_rm) {
968 if (ctxt->modrm_mod != 0)
975 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
976 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
977 ctxt->modrm_seg = VCPU_SREG_SS;
978 modrm_ea = (u16)modrm_ea;
980 /* 32/64-bit ModR/M decode. */
981 if ((ctxt->modrm_rm & 7) == 4) {
982 sib = insn_fetch(u8, ctxt);
983 index_reg |= (sib >> 3) & 7;
987 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
988 modrm_ea += insn_fetch(s32, ctxt);
990 modrm_ea += ctxt->regs[base_reg];
992 modrm_ea += ctxt->regs[index_reg] << scale;
993 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
994 if (ctxt->mode == X86EMUL_MODE_PROT64)
995 ctxt->rip_relative = 1;
997 modrm_ea += ctxt->regs[ctxt->modrm_rm];
998 switch (ctxt->modrm_mod) {
1000 if (ctxt->modrm_rm == 5)
1001 modrm_ea += insn_fetch(s32, ctxt);
1004 modrm_ea += insn_fetch(s8, ctxt);
1007 modrm_ea += insn_fetch(s32, ctxt);
1011 op->addr.mem.ea = modrm_ea;
1016 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1019 int rc = X86EMUL_CONTINUE;
1022 switch (ctxt->ad_bytes) {
1024 op->addr.mem.ea = insn_fetch(u16, ctxt);
1027 op->addr.mem.ea = insn_fetch(u32, ctxt);
1030 op->addr.mem.ea = insn_fetch(u64, ctxt);
1037 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1041 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1042 mask = ~(ctxt->dst.bytes * 8 - 1);
1044 if (ctxt->src.bytes == 2)
1045 sv = (s16)ctxt->src.val & (s16)mask;
1046 else if (ctxt->src.bytes == 4)
1047 sv = (s32)ctxt->src.val & (s32)mask;
1049 ctxt->dst.addr.mem.ea += (sv >> 3);
1052 /* only subword offset */
1053 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1056 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1057 unsigned long addr, void *dest, unsigned size)
1060 struct read_cache *mc = &ctxt->mem_read;
1063 int n = min(size, 8u);
1065 if (mc->pos < mc->end)
1068 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1070 if (rc != X86EMUL_CONTINUE)
1075 memcpy(dest, mc->data + mc->pos, n);
1080 return X86EMUL_CONTINUE;
1083 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1084 struct segmented_address addr,
1091 rc = linearize(ctxt, addr, size, false, &linear);
1092 if (rc != X86EMUL_CONTINUE)
1094 return read_emulated(ctxt, linear, data, size);
1097 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1098 struct segmented_address addr,
1105 rc = linearize(ctxt, addr, size, true, &linear);
1106 if (rc != X86EMUL_CONTINUE)
1108 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1112 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1113 struct segmented_address addr,
1114 const void *orig_data, const void *data,
1120 rc = linearize(ctxt, addr, size, true, &linear);
1121 if (rc != X86EMUL_CONTINUE)
1123 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1124 size, &ctxt->exception);
1127 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1128 unsigned int size, unsigned short port,
1131 struct read_cache *rc = &ctxt->io_read;
1133 if (rc->pos == rc->end) { /* refill pio read ahead */
1134 unsigned int in_page, n;
1135 unsigned int count = ctxt->rep_prefix ?
1136 address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
1137 in_page = (ctxt->eflags & EFLG_DF) ?
1138 offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
1139 PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
1140 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1144 rc->pos = rc->end = 0;
1145 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1150 memcpy(dest, rc->data + rc->pos, size);
1155 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1156 u16 selector, struct desc_ptr *dt)
1158 struct x86_emulate_ops *ops = ctxt->ops;
1160 if (selector & 1 << 2) {
1161 struct desc_struct desc;
1164 memset (dt, 0, sizeof *dt);
1165 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1168 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1169 dt->address = get_desc_base(&desc);
1171 ops->get_gdt(ctxt, dt);
1174 /* allowed just for 8 bytes segments */
1175 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1176 u16 selector, struct desc_struct *desc)
1179 u16 index = selector >> 3;
1182 get_descriptor_table_ptr(ctxt, selector, &dt);
1184 if (dt.size < index * 8 + 7)
1185 return emulate_gp(ctxt, selector & 0xfffc);
1187 addr = dt.address + index * 8;
1188 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1192 /* allowed just for 8 bytes segments */
1193 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1194 u16 selector, struct desc_struct *desc)
1197 u16 index = selector >> 3;
1200 get_descriptor_table_ptr(ctxt, selector, &dt);
1202 if (dt.size < index * 8 + 7)
1203 return emulate_gp(ctxt, selector & 0xfffc);
1205 addr = dt.address + index * 8;
1206 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1210 /* Does not support long mode */
1211 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1212 u16 selector, int seg)
1214 struct desc_struct seg_desc;
1216 unsigned err_vec = GP_VECTOR;
1218 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1221 memset(&seg_desc, 0, sizeof seg_desc);
1223 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1224 || ctxt->mode == X86EMUL_MODE_REAL) {
1225 /* set real mode segment descriptor */
1226 set_desc_base(&seg_desc, selector << 4);
1227 set_desc_limit(&seg_desc, 0xffff);
1234 /* NULL selector is not valid for TR, CS and SS */
1235 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1239 /* TR should be in GDT only */
1240 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1243 if (null_selector) /* for NULL selector skip all following checks */
1246 ret = read_segment_descriptor(ctxt, selector, &seg_desc);
1247 if (ret != X86EMUL_CONTINUE)
1250 err_code = selector & 0xfffc;
1251 err_vec = GP_VECTOR;
1253 /* can't load system descriptor into segment selecor */
1254 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1258 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1264 cpl = ctxt->ops->cpl(ctxt);
1269 * segment is not a writable data segment or segment
1270 * selector's RPL != CPL or segment selector's RPL != CPL
1272 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1276 if (!(seg_desc.type & 8))
1279 if (seg_desc.type & 4) {
1285 if (rpl > cpl || dpl != cpl)
1288 /* CS(RPL) <- CPL */
1289 selector = (selector & 0xfffc) | cpl;
1292 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1295 case VCPU_SREG_LDTR:
1296 if (seg_desc.s || seg_desc.type != 2)
1299 default: /* DS, ES, FS, or GS */
1301 * segment is not a data or readable code segment or
1302 * ((segment is a data or nonconforming code segment)
1303 * and (both RPL and CPL > DPL))
1305 if ((seg_desc.type & 0xa) == 0x8 ||
1306 (((seg_desc.type & 0xc) != 0xc) &&
1307 (rpl > dpl && cpl > dpl)))
1313 /* mark segment as accessed */
1315 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1316 if (ret != X86EMUL_CONTINUE)
1320 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1321 return X86EMUL_CONTINUE;
1323 emulate_exception(ctxt, err_vec, err_code, true);
1324 return X86EMUL_PROPAGATE_FAULT;
1327 static void write_register_operand(struct operand *op)
1329 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1330 switch (op->bytes) {
1332 *(u8 *)op->addr.reg = (u8)op->val;
1335 *(u16 *)op->addr.reg = (u16)op->val;
1338 *op->addr.reg = (u32)op->val;
1339 break; /* 64b: zero-extend */
1341 *op->addr.reg = op->val;
1346 static int writeback(struct x86_emulate_ctxt *ctxt)
1350 switch (ctxt->dst.type) {
1352 write_register_operand(&ctxt->dst);
1355 if (ctxt->lock_prefix)
1356 rc = segmented_cmpxchg(ctxt,
1358 &ctxt->dst.orig_val,
1362 rc = segmented_write(ctxt,
1366 if (rc != X86EMUL_CONTINUE)
1370 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1378 return X86EMUL_CONTINUE;
1381 static int em_push(struct x86_emulate_ctxt *ctxt)
1383 struct segmented_address addr;
1385 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], -ctxt->op_bytes);
1386 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1387 addr.seg = VCPU_SREG_SS;
1389 /* Disable writeback. */
1390 ctxt->dst.type = OP_NONE;
1391 return segmented_write(ctxt, addr, &ctxt->src.val, ctxt->op_bytes);
1394 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1395 void *dest, int len)
1398 struct segmented_address addr;
1400 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1401 addr.seg = VCPU_SREG_SS;
1402 rc = segmented_read(ctxt, addr, dest, len);
1403 if (rc != X86EMUL_CONTINUE)
1406 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], len);
1410 static int em_pop(struct x86_emulate_ctxt *ctxt)
1412 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1415 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1416 void *dest, int len)
1419 unsigned long val, change_mask;
1420 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1421 int cpl = ctxt->ops->cpl(ctxt);
1423 rc = emulate_pop(ctxt, &val, len);
1424 if (rc != X86EMUL_CONTINUE)
1427 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1428 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1430 switch(ctxt->mode) {
1431 case X86EMUL_MODE_PROT64:
1432 case X86EMUL_MODE_PROT32:
1433 case X86EMUL_MODE_PROT16:
1435 change_mask |= EFLG_IOPL;
1437 change_mask |= EFLG_IF;
1439 case X86EMUL_MODE_VM86:
1441 return emulate_gp(ctxt, 0);
1442 change_mask |= EFLG_IF;
1444 default: /* real mode */
1445 change_mask |= (EFLG_IOPL | EFLG_IF);
1449 *(unsigned long *)dest =
1450 (ctxt->eflags & ~change_mask) | (val & change_mask);
1455 static int em_popf(struct x86_emulate_ctxt *ctxt)
1457 ctxt->dst.type = OP_REG;
1458 ctxt->dst.addr.reg = &ctxt->eflags;
1459 ctxt->dst.bytes = ctxt->op_bytes;
1460 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1463 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1465 int seg = ctxt->src2.val;
1467 ctxt->src.val = get_segment_selector(ctxt, seg);
1469 return em_push(ctxt);
1472 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1474 int seg = ctxt->src2.val;
1475 unsigned long selector;
1478 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1479 if (rc != X86EMUL_CONTINUE)
1482 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1486 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1488 unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
1489 int rc = X86EMUL_CONTINUE;
1490 int reg = VCPU_REGS_RAX;
1492 while (reg <= VCPU_REGS_RDI) {
1493 (reg == VCPU_REGS_RSP) ?
1494 (ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
1497 if (rc != X86EMUL_CONTINUE)
1506 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1508 ctxt->src.val = (unsigned long)ctxt->eflags;
1509 return em_push(ctxt);
1512 static int em_popa(struct x86_emulate_ctxt *ctxt)
1514 int rc = X86EMUL_CONTINUE;
1515 int reg = VCPU_REGS_RDI;
1517 while (reg >= VCPU_REGS_RAX) {
1518 if (reg == VCPU_REGS_RSP) {
1519 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP],
1524 rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
1525 if (rc != X86EMUL_CONTINUE)
1532 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1534 struct x86_emulate_ops *ops = ctxt->ops;
1541 /* TODO: Add limit checks */
1542 ctxt->src.val = ctxt->eflags;
1544 if (rc != X86EMUL_CONTINUE)
1547 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1549 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1551 if (rc != X86EMUL_CONTINUE)
1554 ctxt->src.val = ctxt->_eip;
1556 if (rc != X86EMUL_CONTINUE)
1559 ops->get_idt(ctxt, &dt);
1561 eip_addr = dt.address + (irq << 2);
1562 cs_addr = dt.address + (irq << 2) + 2;
1564 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1565 if (rc != X86EMUL_CONTINUE)
1568 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1569 if (rc != X86EMUL_CONTINUE)
1572 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1573 if (rc != X86EMUL_CONTINUE)
1581 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1583 switch(ctxt->mode) {
1584 case X86EMUL_MODE_REAL:
1585 return emulate_int_real(ctxt, irq);
1586 case X86EMUL_MODE_VM86:
1587 case X86EMUL_MODE_PROT16:
1588 case X86EMUL_MODE_PROT32:
1589 case X86EMUL_MODE_PROT64:
1591 /* Protected mode interrupts unimplemented yet */
1592 return X86EMUL_UNHANDLEABLE;
1596 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1598 int rc = X86EMUL_CONTINUE;
1599 unsigned long temp_eip = 0;
1600 unsigned long temp_eflags = 0;
1601 unsigned long cs = 0;
1602 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1603 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1604 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1605 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1607 /* TODO: Add stack limit check */
1609 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1611 if (rc != X86EMUL_CONTINUE)
1614 if (temp_eip & ~0xffff)
1615 return emulate_gp(ctxt, 0);
1617 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1619 if (rc != X86EMUL_CONTINUE)
1622 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1624 if (rc != X86EMUL_CONTINUE)
1627 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1629 if (rc != X86EMUL_CONTINUE)
1632 ctxt->_eip = temp_eip;
1635 if (ctxt->op_bytes == 4)
1636 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1637 else if (ctxt->op_bytes == 2) {
1638 ctxt->eflags &= ~0xffff;
1639 ctxt->eflags |= temp_eflags;
1642 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1643 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1648 static int em_iret(struct x86_emulate_ctxt *ctxt)
1650 switch(ctxt->mode) {
1651 case X86EMUL_MODE_REAL:
1652 return emulate_iret_real(ctxt);
1653 case X86EMUL_MODE_VM86:
1654 case X86EMUL_MODE_PROT16:
1655 case X86EMUL_MODE_PROT32:
1656 case X86EMUL_MODE_PROT64:
1658 /* iret from protected mode unimplemented yet */
1659 return X86EMUL_UNHANDLEABLE;
1663 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1668 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1670 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1671 if (rc != X86EMUL_CONTINUE)
1675 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1676 return X86EMUL_CONTINUE;
1679 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1681 switch (ctxt->modrm_reg) {
1683 emulate_2op_SrcB(ctxt, "rol");
1686 emulate_2op_SrcB(ctxt, "ror");
1689 emulate_2op_SrcB(ctxt, "rcl");
1692 emulate_2op_SrcB(ctxt, "rcr");
1694 case 4: /* sal/shl */
1695 case 6: /* sal/shl */
1696 emulate_2op_SrcB(ctxt, "sal");
1699 emulate_2op_SrcB(ctxt, "shr");
1702 emulate_2op_SrcB(ctxt, "sar");
1705 return X86EMUL_CONTINUE;
1708 static int em_not(struct x86_emulate_ctxt *ctxt)
1710 ctxt->dst.val = ~ctxt->dst.val;
1711 return X86EMUL_CONTINUE;
1714 static int em_neg(struct x86_emulate_ctxt *ctxt)
1716 emulate_1op(ctxt, "neg");
1717 return X86EMUL_CONTINUE;
1720 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
1724 emulate_1op_rax_rdx(ctxt, "mul", ex);
1725 return X86EMUL_CONTINUE;
1728 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
1732 emulate_1op_rax_rdx(ctxt, "imul", ex);
1733 return X86EMUL_CONTINUE;
1736 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1740 emulate_1op_rax_rdx(ctxt, "div", de);
1742 return emulate_de(ctxt);
1743 return X86EMUL_CONTINUE;
1746 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
1750 emulate_1op_rax_rdx(ctxt, "idiv", de);
1752 return emulate_de(ctxt);
1753 return X86EMUL_CONTINUE;
1756 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1758 int rc = X86EMUL_CONTINUE;
1760 switch (ctxt->modrm_reg) {
1762 emulate_1op(ctxt, "inc");
1765 emulate_1op(ctxt, "dec");
1767 case 2: /* call near abs */ {
1769 old_eip = ctxt->_eip;
1770 ctxt->_eip = ctxt->src.val;
1771 ctxt->src.val = old_eip;
1775 case 4: /* jmp abs */
1776 ctxt->_eip = ctxt->src.val;
1778 case 5: /* jmp far */
1779 rc = em_jmp_far(ctxt);
1788 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
1790 u64 old = ctxt->dst.orig_val64;
1792 if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
1793 ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
1794 ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1795 ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1796 ctxt->eflags &= ~EFLG_ZF;
1798 ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
1799 (u32) ctxt->regs[VCPU_REGS_RBX];
1801 ctxt->eflags |= EFLG_ZF;
1803 return X86EMUL_CONTINUE;
1806 static int em_ret(struct x86_emulate_ctxt *ctxt)
1808 ctxt->dst.type = OP_REG;
1809 ctxt->dst.addr.reg = &ctxt->_eip;
1810 ctxt->dst.bytes = ctxt->op_bytes;
1811 return em_pop(ctxt);
1814 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
1819 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
1820 if (rc != X86EMUL_CONTINUE)
1822 if (ctxt->op_bytes == 4)
1823 ctxt->_eip = (u32)ctxt->_eip;
1824 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1825 if (rc != X86EMUL_CONTINUE)
1827 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1831 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
1833 /* Save real source value, then compare EAX against destination. */
1834 ctxt->src.orig_val = ctxt->src.val;
1835 ctxt->src.val = ctxt->regs[VCPU_REGS_RAX];
1836 emulate_2op_SrcV(ctxt, "cmp");
1838 if (ctxt->eflags & EFLG_ZF) {
1839 /* Success: write back to memory. */
1840 ctxt->dst.val = ctxt->src.orig_val;
1842 /* Failure: write the value we saw to EAX. */
1843 ctxt->dst.type = OP_REG;
1844 ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX];
1846 return X86EMUL_CONTINUE;
1849 static int em_lseg(struct x86_emulate_ctxt *ctxt)
1851 int seg = ctxt->src2.val;
1855 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1857 rc = load_segment_descriptor(ctxt, sel, seg);
1858 if (rc != X86EMUL_CONTINUE)
1861 ctxt->dst.val = ctxt->src.val;
1866 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1867 struct desc_struct *cs, struct desc_struct *ss)
1871 memset(cs, 0, sizeof(struct desc_struct));
1872 ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1873 memset(ss, 0, sizeof(struct desc_struct));
1875 cs->l = 0; /* will be adjusted later */
1876 set_desc_base(cs, 0); /* flat segment */
1877 cs->g = 1; /* 4kb granularity */
1878 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1879 cs->type = 0x0b; /* Read, Execute, Accessed */
1881 cs->dpl = 0; /* will be adjusted later */
1885 set_desc_base(ss, 0); /* flat segment */
1886 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1887 ss->g = 1; /* 4kb granularity */
1889 ss->type = 0x03; /* Read/Write, Accessed */
1890 ss->d = 1; /* 32bit stack segment */
1895 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
1897 u32 eax, ebx, ecx, edx;
1900 return ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx)
1901 && ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
1902 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
1903 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
1906 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
1908 struct x86_emulate_ops *ops = ctxt->ops;
1909 u32 eax, ebx, ecx, edx;
1912 * syscall should always be enabled in longmode - so only become
1913 * vendor specific (cpuid) if other modes are active...
1915 if (ctxt->mode == X86EMUL_MODE_PROT64)
1920 if (ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx)) {
1922 * Intel ("GenuineIntel")
1923 * remark: Intel CPUs only support "syscall" in 64bit
1924 * longmode. Also an 64bit guest with a
1925 * 32bit compat-app running will #UD !! While this
1926 * behaviour can be fixed (by emulating) into AMD
1927 * response - CPUs of AMD can't behave like Intel.
1929 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
1930 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
1931 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
1934 /* AMD ("AuthenticAMD") */
1935 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
1936 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
1937 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
1940 /* AMD ("AMDisbetter!") */
1941 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
1942 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
1943 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
1947 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
1951 static int em_syscall(struct x86_emulate_ctxt *ctxt)
1953 struct x86_emulate_ops *ops = ctxt->ops;
1954 struct desc_struct cs, ss;
1959 /* syscall is not available in real mode */
1960 if (ctxt->mode == X86EMUL_MODE_REAL ||
1961 ctxt->mode == X86EMUL_MODE_VM86)
1962 return emulate_ud(ctxt);
1964 if (!(em_syscall_is_enabled(ctxt)))
1965 return emulate_ud(ctxt);
1967 ops->get_msr(ctxt, MSR_EFER, &efer);
1968 setup_syscalls_segments(ctxt, &cs, &ss);
1970 if (!(efer & EFER_SCE))
1971 return emulate_ud(ctxt);
1973 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1975 cs_sel = (u16)(msr_data & 0xfffc);
1976 ss_sel = (u16)(msr_data + 8);
1978 if (efer & EFER_LMA) {
1982 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1983 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1985 ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
1986 if (efer & EFER_LMA) {
1987 #ifdef CONFIG_X86_64
1988 ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1991 ctxt->mode == X86EMUL_MODE_PROT64 ?
1992 MSR_LSTAR : MSR_CSTAR, &msr_data);
1993 ctxt->_eip = msr_data;
1995 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1996 ctxt->eflags &= ~(msr_data | EFLG_RF);
2000 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2001 ctxt->_eip = (u32)msr_data;
2003 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2006 return X86EMUL_CONTINUE;
2009 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2011 struct x86_emulate_ops *ops = ctxt->ops;
2012 struct desc_struct cs, ss;
2017 ops->get_msr(ctxt, MSR_EFER, &efer);
2018 /* inject #GP if in real mode */
2019 if (ctxt->mode == X86EMUL_MODE_REAL)
2020 return emulate_gp(ctxt, 0);
2023 * Not recognized on AMD in compat mode (but is recognized in legacy
2026 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2027 && !vendor_intel(ctxt))
2028 return emulate_ud(ctxt);
2030 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2031 * Therefore, we inject an #UD.
2033 if (ctxt->mode == X86EMUL_MODE_PROT64)
2034 return emulate_ud(ctxt);
2036 setup_syscalls_segments(ctxt, &cs, &ss);
2038 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2039 switch (ctxt->mode) {
2040 case X86EMUL_MODE_PROT32:
2041 if ((msr_data & 0xfffc) == 0x0)
2042 return emulate_gp(ctxt, 0);
2044 case X86EMUL_MODE_PROT64:
2045 if (msr_data == 0x0)
2046 return emulate_gp(ctxt, 0);
2050 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2051 cs_sel = (u16)msr_data;
2052 cs_sel &= ~SELECTOR_RPL_MASK;
2053 ss_sel = cs_sel + 8;
2054 ss_sel &= ~SELECTOR_RPL_MASK;
2055 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2060 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2061 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2063 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2064 ctxt->_eip = msr_data;
2066 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2067 ctxt->regs[VCPU_REGS_RSP] = msr_data;
2069 return X86EMUL_CONTINUE;
2072 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2074 struct x86_emulate_ops *ops = ctxt->ops;
2075 struct desc_struct cs, ss;
2078 u16 cs_sel = 0, ss_sel = 0;
2080 /* inject #GP if in real mode or Virtual 8086 mode */
2081 if (ctxt->mode == X86EMUL_MODE_REAL ||
2082 ctxt->mode == X86EMUL_MODE_VM86)
2083 return emulate_gp(ctxt, 0);
2085 setup_syscalls_segments(ctxt, &cs, &ss);
2087 if ((ctxt->rex_prefix & 0x8) != 0x0)
2088 usermode = X86EMUL_MODE_PROT64;
2090 usermode = X86EMUL_MODE_PROT32;
2094 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2096 case X86EMUL_MODE_PROT32:
2097 cs_sel = (u16)(msr_data + 16);
2098 if ((msr_data & 0xfffc) == 0x0)
2099 return emulate_gp(ctxt, 0);
2100 ss_sel = (u16)(msr_data + 24);
2102 case X86EMUL_MODE_PROT64:
2103 cs_sel = (u16)(msr_data + 32);
2104 if (msr_data == 0x0)
2105 return emulate_gp(ctxt, 0);
2106 ss_sel = cs_sel + 8;
2111 cs_sel |= SELECTOR_RPL_MASK;
2112 ss_sel |= SELECTOR_RPL_MASK;
2114 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2115 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2117 ctxt->_eip = ctxt->regs[VCPU_REGS_RDX];
2118 ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX];
2120 return X86EMUL_CONTINUE;
2123 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2126 if (ctxt->mode == X86EMUL_MODE_REAL)
2128 if (ctxt->mode == X86EMUL_MODE_VM86)
2130 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2131 return ctxt->ops->cpl(ctxt) > iopl;
2134 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2137 struct x86_emulate_ops *ops = ctxt->ops;
2138 struct desc_struct tr_seg;
2141 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2142 unsigned mask = (1 << len) - 1;
2145 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2148 if (desc_limit_scaled(&tr_seg) < 103)
2150 base = get_desc_base(&tr_seg);
2151 #ifdef CONFIG_X86_64
2152 base |= ((u64)base3) << 32;
2154 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2155 if (r != X86EMUL_CONTINUE)
2157 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2159 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2160 if (r != X86EMUL_CONTINUE)
2162 if ((perm >> bit_idx) & mask)
2167 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2173 if (emulator_bad_iopl(ctxt))
2174 if (!emulator_io_port_access_allowed(ctxt, port, len))
2177 ctxt->perm_ok = true;
2182 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2183 struct tss_segment_16 *tss)
2185 tss->ip = ctxt->_eip;
2186 tss->flag = ctxt->eflags;
2187 tss->ax = ctxt->regs[VCPU_REGS_RAX];
2188 tss->cx = ctxt->regs[VCPU_REGS_RCX];
2189 tss->dx = ctxt->regs[VCPU_REGS_RDX];
2190 tss->bx = ctxt->regs[VCPU_REGS_RBX];
2191 tss->sp = ctxt->regs[VCPU_REGS_RSP];
2192 tss->bp = ctxt->regs[VCPU_REGS_RBP];
2193 tss->si = ctxt->regs[VCPU_REGS_RSI];
2194 tss->di = ctxt->regs[VCPU_REGS_RDI];
2196 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2197 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2198 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2199 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2200 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2203 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2204 struct tss_segment_16 *tss)
2208 ctxt->_eip = tss->ip;
2209 ctxt->eflags = tss->flag | 2;
2210 ctxt->regs[VCPU_REGS_RAX] = tss->ax;
2211 ctxt->regs[VCPU_REGS_RCX] = tss->cx;
2212 ctxt->regs[VCPU_REGS_RDX] = tss->dx;
2213 ctxt->regs[VCPU_REGS_RBX] = tss->bx;
2214 ctxt->regs[VCPU_REGS_RSP] = tss->sp;
2215 ctxt->regs[VCPU_REGS_RBP] = tss->bp;
2216 ctxt->regs[VCPU_REGS_RSI] = tss->si;
2217 ctxt->regs[VCPU_REGS_RDI] = tss->di;
2220 * SDM says that segment selectors are loaded before segment
2223 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2224 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2225 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2226 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2227 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2230 * Now load segment descriptors. If fault happenes at this stage
2231 * it is handled in a context of new task
2233 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2234 if (ret != X86EMUL_CONTINUE)
2236 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2237 if (ret != X86EMUL_CONTINUE)
2239 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2240 if (ret != X86EMUL_CONTINUE)
2242 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2243 if (ret != X86EMUL_CONTINUE)
2245 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2246 if (ret != X86EMUL_CONTINUE)
2249 return X86EMUL_CONTINUE;
2252 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2253 u16 tss_selector, u16 old_tss_sel,
2254 ulong old_tss_base, struct desc_struct *new_desc)
2256 struct x86_emulate_ops *ops = ctxt->ops;
2257 struct tss_segment_16 tss_seg;
2259 u32 new_tss_base = get_desc_base(new_desc);
2261 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2263 if (ret != X86EMUL_CONTINUE)
2264 /* FIXME: need to provide precise fault address */
2267 save_state_to_tss16(ctxt, &tss_seg);
2269 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2271 if (ret != X86EMUL_CONTINUE)
2272 /* FIXME: need to provide precise fault address */
2275 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2277 if (ret != X86EMUL_CONTINUE)
2278 /* FIXME: need to provide precise fault address */
2281 if (old_tss_sel != 0xffff) {
2282 tss_seg.prev_task_link = old_tss_sel;
2284 ret = ops->write_std(ctxt, new_tss_base,
2285 &tss_seg.prev_task_link,
2286 sizeof tss_seg.prev_task_link,
2288 if (ret != X86EMUL_CONTINUE)
2289 /* FIXME: need to provide precise fault address */
2293 return load_state_from_tss16(ctxt, &tss_seg);
2296 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2297 struct tss_segment_32 *tss)
2299 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2300 tss->eip = ctxt->_eip;
2301 tss->eflags = ctxt->eflags;
2302 tss->eax = ctxt->regs[VCPU_REGS_RAX];
2303 tss->ecx = ctxt->regs[VCPU_REGS_RCX];
2304 tss->edx = ctxt->regs[VCPU_REGS_RDX];
2305 tss->ebx = ctxt->regs[VCPU_REGS_RBX];
2306 tss->esp = ctxt->regs[VCPU_REGS_RSP];
2307 tss->ebp = ctxt->regs[VCPU_REGS_RBP];
2308 tss->esi = ctxt->regs[VCPU_REGS_RSI];
2309 tss->edi = ctxt->regs[VCPU_REGS_RDI];
2311 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2312 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2313 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2314 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2315 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2316 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2317 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2320 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2321 struct tss_segment_32 *tss)
2325 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2326 return emulate_gp(ctxt, 0);
2327 ctxt->_eip = tss->eip;
2328 ctxt->eflags = tss->eflags | 2;
2329 ctxt->regs[VCPU_REGS_RAX] = tss->eax;
2330 ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
2331 ctxt->regs[VCPU_REGS_RDX] = tss->edx;
2332 ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
2333 ctxt->regs[VCPU_REGS_RSP] = tss->esp;
2334 ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
2335 ctxt->regs[VCPU_REGS_RSI] = tss->esi;
2336 ctxt->regs[VCPU_REGS_RDI] = tss->edi;
2339 * SDM says that segment selectors are loaded before segment
2342 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2343 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2344 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2345 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2346 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2347 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2348 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2351 * Now load segment descriptors. If fault happenes at this stage
2352 * it is handled in a context of new task
2354 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2355 if (ret != X86EMUL_CONTINUE)
2357 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2358 if (ret != X86EMUL_CONTINUE)
2360 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2361 if (ret != X86EMUL_CONTINUE)
2363 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2364 if (ret != X86EMUL_CONTINUE)
2366 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2367 if (ret != X86EMUL_CONTINUE)
2369 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2370 if (ret != X86EMUL_CONTINUE)
2372 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2373 if (ret != X86EMUL_CONTINUE)
2376 return X86EMUL_CONTINUE;
2379 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2380 u16 tss_selector, u16 old_tss_sel,
2381 ulong old_tss_base, struct desc_struct *new_desc)
2383 struct x86_emulate_ops *ops = ctxt->ops;
2384 struct tss_segment_32 tss_seg;
2386 u32 new_tss_base = get_desc_base(new_desc);
2388 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2390 if (ret != X86EMUL_CONTINUE)
2391 /* FIXME: need to provide precise fault address */
2394 save_state_to_tss32(ctxt, &tss_seg);
2396 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2398 if (ret != X86EMUL_CONTINUE)
2399 /* FIXME: need to provide precise fault address */
2402 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2404 if (ret != X86EMUL_CONTINUE)
2405 /* FIXME: need to provide precise fault address */
2408 if (old_tss_sel != 0xffff) {
2409 tss_seg.prev_task_link = old_tss_sel;
2411 ret = ops->write_std(ctxt, new_tss_base,
2412 &tss_seg.prev_task_link,
2413 sizeof tss_seg.prev_task_link,
2415 if (ret != X86EMUL_CONTINUE)
2416 /* FIXME: need to provide precise fault address */
2420 return load_state_from_tss32(ctxt, &tss_seg);
2423 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2424 u16 tss_selector, int reason,
2425 bool has_error_code, u32 error_code)
2427 struct x86_emulate_ops *ops = ctxt->ops;
2428 struct desc_struct curr_tss_desc, next_tss_desc;
2430 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2431 ulong old_tss_base =
2432 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2435 /* FIXME: old_tss_base == ~0 ? */
2437 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2438 if (ret != X86EMUL_CONTINUE)
2440 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2441 if (ret != X86EMUL_CONTINUE)
2444 /* FIXME: check that next_tss_desc is tss */
2446 if (reason != TASK_SWITCH_IRET) {
2447 if ((tss_selector & 3) > next_tss_desc.dpl ||
2448 ops->cpl(ctxt) > next_tss_desc.dpl)
2449 return emulate_gp(ctxt, 0);
2452 desc_limit = desc_limit_scaled(&next_tss_desc);
2453 if (!next_tss_desc.p ||
2454 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2455 desc_limit < 0x2b)) {
2456 emulate_ts(ctxt, tss_selector & 0xfffc);
2457 return X86EMUL_PROPAGATE_FAULT;
2460 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2461 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2462 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2465 if (reason == TASK_SWITCH_IRET)
2466 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2468 /* set back link to prev task only if NT bit is set in eflags
2469 note that old_tss_sel is not used afetr this point */
2470 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2471 old_tss_sel = 0xffff;
2473 if (next_tss_desc.type & 8)
2474 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2475 old_tss_base, &next_tss_desc);
2477 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2478 old_tss_base, &next_tss_desc);
2479 if (ret != X86EMUL_CONTINUE)
2482 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2483 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2485 if (reason != TASK_SWITCH_IRET) {
2486 next_tss_desc.type |= (1 << 1); /* set busy flag */
2487 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2490 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2491 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2493 if (has_error_code) {
2494 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2495 ctxt->lock_prefix = 0;
2496 ctxt->src.val = (unsigned long) error_code;
2497 ret = em_push(ctxt);
2503 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2504 u16 tss_selector, int reason,
2505 bool has_error_code, u32 error_code)
2509 ctxt->_eip = ctxt->eip;
2510 ctxt->dst.type = OP_NONE;
2512 rc = emulator_do_task_switch(ctxt, tss_selector, reason,
2513 has_error_code, error_code);
2515 if (rc == X86EMUL_CONTINUE)
2516 ctxt->eip = ctxt->_eip;
2518 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2521 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2522 int reg, struct operand *op)
2524 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2526 register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
2527 op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
2528 op->addr.mem.seg = seg;
2531 static int em_das(struct x86_emulate_ctxt *ctxt)
2534 bool af, cf, old_cf;
2536 cf = ctxt->eflags & X86_EFLAGS_CF;
2542 af = ctxt->eflags & X86_EFLAGS_AF;
2543 if ((al & 0x0f) > 9 || af) {
2545 cf = old_cf | (al >= 250);
2550 if (old_al > 0x99 || old_cf) {
2556 /* Set PF, ZF, SF */
2557 ctxt->src.type = OP_IMM;
2559 ctxt->src.bytes = 1;
2560 emulate_2op_SrcV(ctxt, "or");
2561 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2563 ctxt->eflags |= X86_EFLAGS_CF;
2565 ctxt->eflags |= X86_EFLAGS_AF;
2566 return X86EMUL_CONTINUE;
2569 static int em_call(struct x86_emulate_ctxt *ctxt)
2571 long rel = ctxt->src.val;
2573 ctxt->src.val = (unsigned long)ctxt->_eip;
2575 return em_push(ctxt);
2578 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2584 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2585 old_eip = ctxt->_eip;
2587 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2588 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2589 return X86EMUL_CONTINUE;
2592 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2594 ctxt->src.val = old_cs;
2596 if (rc != X86EMUL_CONTINUE)
2599 ctxt->src.val = old_eip;
2600 return em_push(ctxt);
2603 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2607 ctxt->dst.type = OP_REG;
2608 ctxt->dst.addr.reg = &ctxt->_eip;
2609 ctxt->dst.bytes = ctxt->op_bytes;
2610 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2611 if (rc != X86EMUL_CONTINUE)
2613 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], ctxt->src.val);
2614 return X86EMUL_CONTINUE;
2617 static int em_add(struct x86_emulate_ctxt *ctxt)
2619 emulate_2op_SrcV(ctxt, "add");
2620 return X86EMUL_CONTINUE;
2623 static int em_or(struct x86_emulate_ctxt *ctxt)
2625 emulate_2op_SrcV(ctxt, "or");
2626 return X86EMUL_CONTINUE;
2629 static int em_adc(struct x86_emulate_ctxt *ctxt)
2631 emulate_2op_SrcV(ctxt, "adc");
2632 return X86EMUL_CONTINUE;
2635 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2637 emulate_2op_SrcV(ctxt, "sbb");
2638 return X86EMUL_CONTINUE;
2641 static int em_and(struct x86_emulate_ctxt *ctxt)
2643 emulate_2op_SrcV(ctxt, "and");
2644 return X86EMUL_CONTINUE;
2647 static int em_sub(struct x86_emulate_ctxt *ctxt)
2649 emulate_2op_SrcV(ctxt, "sub");
2650 return X86EMUL_CONTINUE;
2653 static int em_xor(struct x86_emulate_ctxt *ctxt)
2655 emulate_2op_SrcV(ctxt, "xor");
2656 return X86EMUL_CONTINUE;
2659 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2661 emulate_2op_SrcV(ctxt, "cmp");
2662 /* Disable writeback. */
2663 ctxt->dst.type = OP_NONE;
2664 return X86EMUL_CONTINUE;
2667 static int em_test(struct x86_emulate_ctxt *ctxt)
2669 emulate_2op_SrcV(ctxt, "test");
2670 /* Disable writeback. */
2671 ctxt->dst.type = OP_NONE;
2672 return X86EMUL_CONTINUE;
2675 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2677 /* Write back the register source. */
2678 ctxt->src.val = ctxt->dst.val;
2679 write_register_operand(&ctxt->src);
2681 /* Write back the memory destination with implicit LOCK prefix. */
2682 ctxt->dst.val = ctxt->src.orig_val;
2683 ctxt->lock_prefix = 1;
2684 return X86EMUL_CONTINUE;
2687 static int em_imul(struct x86_emulate_ctxt *ctxt)
2689 emulate_2op_SrcV_nobyte(ctxt, "imul");
2690 return X86EMUL_CONTINUE;
2693 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2695 ctxt->dst.val = ctxt->src2.val;
2696 return em_imul(ctxt);
2699 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2701 ctxt->dst.type = OP_REG;
2702 ctxt->dst.bytes = ctxt->src.bytes;
2703 ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
2704 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2706 return X86EMUL_CONTINUE;
2709 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2713 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2714 ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
2715 ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
2716 return X86EMUL_CONTINUE;
2719 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
2723 if (ctxt->ops->read_pmc(ctxt, ctxt->regs[VCPU_REGS_RCX], &pmc))
2724 return emulate_gp(ctxt, 0);
2725 ctxt->regs[VCPU_REGS_RAX] = (u32)pmc;
2726 ctxt->regs[VCPU_REGS_RDX] = pmc >> 32;
2727 return X86EMUL_CONTINUE;
2730 static int em_mov(struct x86_emulate_ctxt *ctxt)
2732 ctxt->dst.val = ctxt->src.val;
2733 return X86EMUL_CONTINUE;
2736 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
2738 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
2739 return emulate_gp(ctxt, 0);
2741 /* Disable writeback. */
2742 ctxt->dst.type = OP_NONE;
2743 return X86EMUL_CONTINUE;
2746 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
2750 if (ctxt->mode == X86EMUL_MODE_PROT64)
2751 val = ctxt->src.val & ~0ULL;
2753 val = ctxt->src.val & ~0U;
2755 /* #UD condition is already handled. */
2756 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
2757 return emulate_gp(ctxt, 0);
2759 /* Disable writeback. */
2760 ctxt->dst.type = OP_NONE;
2761 return X86EMUL_CONTINUE;
2764 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
2768 msr_data = (u32)ctxt->regs[VCPU_REGS_RAX]
2769 | ((u64)ctxt->regs[VCPU_REGS_RDX] << 32);
2770 if (ctxt->ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data))
2771 return emulate_gp(ctxt, 0);
2773 return X86EMUL_CONTINUE;
2776 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
2780 if (ctxt->ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data))
2781 return emulate_gp(ctxt, 0);
2783 ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data;
2784 ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32;
2785 return X86EMUL_CONTINUE;
2788 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
2790 if (ctxt->modrm_reg > VCPU_SREG_GS)
2791 return emulate_ud(ctxt);
2793 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
2794 return X86EMUL_CONTINUE;
2797 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
2799 u16 sel = ctxt->src.val;
2801 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
2802 return emulate_ud(ctxt);
2804 if (ctxt->modrm_reg == VCPU_SREG_SS)
2805 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2807 /* Disable writeback. */
2808 ctxt->dst.type = OP_NONE;
2809 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
2812 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2814 memcpy(&ctxt->dst.vec_val, &ctxt->src.vec_val, ctxt->op_bytes);
2815 return X86EMUL_CONTINUE;
2818 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2823 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
2824 if (rc == X86EMUL_CONTINUE)
2825 ctxt->ops->invlpg(ctxt, linear);
2826 /* Disable writeback. */
2827 ctxt->dst.type = OP_NONE;
2828 return X86EMUL_CONTINUE;
2831 static int em_clts(struct x86_emulate_ctxt *ctxt)
2835 cr0 = ctxt->ops->get_cr(ctxt, 0);
2837 ctxt->ops->set_cr(ctxt, 0, cr0);
2838 return X86EMUL_CONTINUE;
2841 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
2845 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
2846 return X86EMUL_UNHANDLEABLE;
2848 rc = ctxt->ops->fix_hypercall(ctxt);
2849 if (rc != X86EMUL_CONTINUE)
2852 /* Let the processor re-execute the fixed hypercall */
2853 ctxt->_eip = ctxt->eip;
2854 /* Disable writeback. */
2855 ctxt->dst.type = OP_NONE;
2856 return X86EMUL_CONTINUE;
2859 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
2861 struct desc_ptr desc_ptr;
2864 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2865 &desc_ptr.size, &desc_ptr.address,
2867 if (rc != X86EMUL_CONTINUE)
2869 ctxt->ops->set_gdt(ctxt, &desc_ptr);
2870 /* Disable writeback. */
2871 ctxt->dst.type = OP_NONE;
2872 return X86EMUL_CONTINUE;
2875 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2879 rc = ctxt->ops->fix_hypercall(ctxt);
2881 /* Disable writeback. */
2882 ctxt->dst.type = OP_NONE;
2886 static int em_lidt(struct x86_emulate_ctxt *ctxt)
2888 struct desc_ptr desc_ptr;
2891 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2892 &desc_ptr.size, &desc_ptr.address,
2894 if (rc != X86EMUL_CONTINUE)
2896 ctxt->ops->set_idt(ctxt, &desc_ptr);
2897 /* Disable writeback. */
2898 ctxt->dst.type = OP_NONE;
2899 return X86EMUL_CONTINUE;
2902 static int em_smsw(struct x86_emulate_ctxt *ctxt)
2904 ctxt->dst.bytes = 2;
2905 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
2906 return X86EMUL_CONTINUE;
2909 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
2911 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2912 | (ctxt->src.val & 0x0f));
2913 ctxt->dst.type = OP_NONE;
2914 return X86EMUL_CONTINUE;
2917 static int em_loop(struct x86_emulate_ctxt *ctxt)
2919 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
2920 if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
2921 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
2922 jmp_rel(ctxt, ctxt->src.val);
2924 return X86EMUL_CONTINUE;
2927 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
2929 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
2930 jmp_rel(ctxt, ctxt->src.val);
2932 return X86EMUL_CONTINUE;
2935 static int em_in(struct x86_emulate_ctxt *ctxt)
2937 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
2939 return X86EMUL_IO_NEEDED;
2941 return X86EMUL_CONTINUE;
2944 static int em_out(struct x86_emulate_ctxt *ctxt)
2946 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
2948 /* Disable writeback. */
2949 ctxt->dst.type = OP_NONE;
2950 return X86EMUL_CONTINUE;
2953 static int em_cli(struct x86_emulate_ctxt *ctxt)
2955 if (emulator_bad_iopl(ctxt))
2956 return emulate_gp(ctxt, 0);
2958 ctxt->eflags &= ~X86_EFLAGS_IF;
2959 return X86EMUL_CONTINUE;
2962 static int em_sti(struct x86_emulate_ctxt *ctxt)
2964 if (emulator_bad_iopl(ctxt))
2965 return emulate_gp(ctxt, 0);
2967 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
2968 ctxt->eflags |= X86_EFLAGS_IF;
2969 return X86EMUL_CONTINUE;
2972 static int em_bt(struct x86_emulate_ctxt *ctxt)
2974 /* Disable writeback. */
2975 ctxt->dst.type = OP_NONE;
2976 /* only subword offset */
2977 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
2979 emulate_2op_SrcV_nobyte(ctxt, "bt");
2980 return X86EMUL_CONTINUE;
2983 static int em_bts(struct x86_emulate_ctxt *ctxt)
2985 emulate_2op_SrcV_nobyte(ctxt, "bts");
2986 return X86EMUL_CONTINUE;
2989 static int em_btr(struct x86_emulate_ctxt *ctxt)
2991 emulate_2op_SrcV_nobyte(ctxt, "btr");
2992 return X86EMUL_CONTINUE;
2995 static int em_btc(struct x86_emulate_ctxt *ctxt)
2997 emulate_2op_SrcV_nobyte(ctxt, "btc");
2998 return X86EMUL_CONTINUE;
3001 static int em_bsf(struct x86_emulate_ctxt *ctxt)
3005 __asm__ ("bsf %2, %0; setz %1"
3006 : "=r"(ctxt->dst.val), "=q"(zf)
3007 : "r"(ctxt->src.val));
3009 ctxt->eflags &= ~X86_EFLAGS_ZF;
3011 ctxt->eflags |= X86_EFLAGS_ZF;
3012 /* Disable writeback. */
3013 ctxt->dst.type = OP_NONE;
3015 return X86EMUL_CONTINUE;
3018 static int em_bsr(struct x86_emulate_ctxt *ctxt)
3022 __asm__ ("bsr %2, %0; setz %1"
3023 : "=r"(ctxt->dst.val), "=q"(zf)
3024 : "r"(ctxt->src.val));
3026 ctxt->eflags &= ~X86_EFLAGS_ZF;
3028 ctxt->eflags |= X86_EFLAGS_ZF;
3029 /* Disable writeback. */
3030 ctxt->dst.type = OP_NONE;
3032 return X86EMUL_CONTINUE;
3035 static bool valid_cr(int nr)
3047 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3049 if (!valid_cr(ctxt->modrm_reg))
3050 return emulate_ud(ctxt);
3052 return X86EMUL_CONTINUE;
3055 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3057 u64 new_val = ctxt->src.val64;
3058 int cr = ctxt->modrm_reg;
3061 static u64 cr_reserved_bits[] = {
3062 0xffffffff00000000ULL,
3063 0, 0, 0, /* CR3 checked later */
3070 return emulate_ud(ctxt);
3072 if (new_val & cr_reserved_bits[cr])
3073 return emulate_gp(ctxt, 0);
3078 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3079 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3080 return emulate_gp(ctxt, 0);
3082 cr4 = ctxt->ops->get_cr(ctxt, 4);
3083 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3085 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3086 !(cr4 & X86_CR4_PAE))
3087 return emulate_gp(ctxt, 0);
3094 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3095 if (efer & EFER_LMA)
3096 rsvd = CR3_L_MODE_RESERVED_BITS;
3097 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3098 rsvd = CR3_PAE_RESERVED_BITS;
3099 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3100 rsvd = CR3_NONPAE_RESERVED_BITS;
3103 return emulate_gp(ctxt, 0);
3108 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3110 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3111 return emulate_gp(ctxt, 0);
3117 return X86EMUL_CONTINUE;
3120 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3124 ctxt->ops->get_dr(ctxt, 7, &dr7);
3126 /* Check if DR7.Global_Enable is set */
3127 return dr7 & (1 << 13);
3130 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3132 int dr = ctxt->modrm_reg;
3136 return emulate_ud(ctxt);
3138 cr4 = ctxt->ops->get_cr(ctxt, 4);
3139 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3140 return emulate_ud(ctxt);
3142 if (check_dr7_gd(ctxt))
3143 return emulate_db(ctxt);
3145 return X86EMUL_CONTINUE;
3148 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3150 u64 new_val = ctxt->src.val64;
3151 int dr = ctxt->modrm_reg;
3153 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3154 return emulate_gp(ctxt, 0);
3156 return check_dr_read(ctxt);
3159 static int check_svme(struct x86_emulate_ctxt *ctxt)
3163 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3165 if (!(efer & EFER_SVME))
3166 return emulate_ud(ctxt);
3168 return X86EMUL_CONTINUE;
3171 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3173 u64 rax = ctxt->regs[VCPU_REGS_RAX];
3175 /* Valid physical address? */
3176 if (rax & 0xffff000000000000ULL)
3177 return emulate_gp(ctxt, 0);
3179 return check_svme(ctxt);
3182 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3184 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3186 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3187 return emulate_ud(ctxt);
3189 return X86EMUL_CONTINUE;
3192 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3194 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3195 u64 rcx = ctxt->regs[VCPU_REGS_RCX];
3197 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3199 return emulate_gp(ctxt, 0);
3201 return X86EMUL_CONTINUE;
3204 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3206 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3207 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3208 return emulate_gp(ctxt, 0);
3210 return X86EMUL_CONTINUE;
3213 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3215 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3216 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3217 return emulate_gp(ctxt, 0);
3219 return X86EMUL_CONTINUE;
3222 #define D(_y) { .flags = (_y) }
3223 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3224 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3225 .check_perm = (_p) }
3227 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3228 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
3229 #define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
3230 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3231 #define II(_f, _e, _i) \
3232 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3233 #define IIP(_f, _e, _i, _p) \
3234 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3235 .check_perm = (_p) }
3236 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3238 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3239 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3240 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3241 #define I2bvIP(_f, _e, _i, _p) \
3242 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3244 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3245 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3246 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3248 static struct opcode group7_rm1[] = {
3249 DI(SrcNone | ModRM | Priv, monitor),
3250 DI(SrcNone | ModRM | Priv, mwait),
3254 static struct opcode group7_rm3[] = {
3255 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
3256 II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
3257 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
3258 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
3259 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
3260 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
3261 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
3262 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
3265 static struct opcode group7_rm7[] = {
3267 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
3271 static struct opcode group1[] = {
3273 I(Lock | PageTable, em_or),
3276 I(Lock | PageTable, em_and),
3282 static struct opcode group1A[] = {
3283 I(DstMem | SrcNone | ModRM | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3286 static struct opcode group3[] = {
3287 I(DstMem | SrcImm | ModRM, em_test),
3288 I(DstMem | SrcImm | ModRM, em_test),
3289 I(DstMem | SrcNone | ModRM | Lock, em_not),
3290 I(DstMem | SrcNone | ModRM | Lock, em_neg),
3291 I(SrcMem | ModRM, em_mul_ex),
3292 I(SrcMem | ModRM, em_imul_ex),
3293 I(SrcMem | ModRM, em_div_ex),
3294 I(SrcMem | ModRM, em_idiv_ex),
3297 static struct opcode group4[] = {
3298 I(ByteOp | DstMem | SrcNone | ModRM | Lock, em_grp45),
3299 I(ByteOp | DstMem | SrcNone | ModRM | Lock, em_grp45),
3303 static struct opcode group5[] = {
3304 I(DstMem | SrcNone | ModRM | Lock, em_grp45),
3305 I(DstMem | SrcNone | ModRM | Lock, em_grp45),
3306 I(SrcMem | ModRM | Stack, em_grp45),
3307 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3308 I(SrcMem | ModRM | Stack, em_grp45),
3309 I(SrcMemFAddr | ModRM | ImplicitOps, em_grp45),
3310 I(SrcMem | ModRM | Stack, em_grp45), N,
3313 static struct opcode group6[] = {
3314 DI(ModRM | Prot, sldt),
3315 DI(ModRM | Prot, str),
3316 DI(ModRM | Prot | Priv, lldt),
3317 DI(ModRM | Prot | Priv, ltr),
3321 static struct group_dual group7 = { {
3322 DI(ModRM | Mov | DstMem | Priv, sgdt),
3323 DI(ModRM | Mov | DstMem | Priv, sidt),
3324 II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
3325 II(ModRM | SrcMem | Priv, em_lidt, lidt),
3326 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3327 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
3328 II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3330 I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
3332 N, EXT(0, group7_rm3),
3333 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3334 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3337 static struct opcode group8[] = {
3339 I(DstMem | SrcImmByte | ModRM, em_bt),
3340 I(DstMem | SrcImmByte | ModRM | Lock | PageTable, em_bts),
3341 I(DstMem | SrcImmByte | ModRM | Lock, em_btr),
3342 I(DstMem | SrcImmByte | ModRM | Lock | PageTable, em_btc),
3345 static struct group_dual group9 = { {
3346 N, I(DstMem64 | ModRM | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3348 N, N, N, N, N, N, N, N,
3351 static struct opcode group11[] = {
3352 I(DstMem | SrcImm | ModRM | Mov | PageTable, em_mov),
3356 static struct gprefix pfx_0f_6f_0f_7f = {
3357 N, N, N, I(Sse, em_movdqu),
3360 static struct opcode opcode_table[256] = {
3362 I6ALU(Lock, em_add),
3363 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3364 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3366 I6ALU(Lock | PageTable, em_or),
3367 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3370 I6ALU(Lock, em_adc),
3371 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3372 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3374 I6ALU(Lock, em_sbb),
3375 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3376 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3378 I6ALU(Lock | PageTable, em_and), N, N,
3380 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3382 I6ALU(Lock, em_xor), N, N,
3384 I6ALU(0, em_cmp), N, N,
3388 X8(I(SrcReg | Stack, em_push)),
3390 X8(I(DstReg | Stack, em_pop)),
3392 I(ImplicitOps | Stack | No64, em_pusha),
3393 I(ImplicitOps | Stack | No64, em_popa),
3394 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3397 I(SrcImm | Mov | Stack, em_push),
3398 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3399 I(SrcImmByte | Mov | Stack, em_push),
3400 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3401 I2bvIP(DstDI | SrcDX | Mov | String, em_in, ins, check_perm_in), /* insb, insw/insd */
3402 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3406 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
3407 G(DstMem | SrcImm | ModRM | Group, group1),
3408 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
3409 G(DstMem | SrcImmByte | ModRM | Group, group1),
3410 I2bv(DstMem | SrcReg | ModRM, em_test),
3411 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3413 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3414 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3415 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3416 D(ModRM | SrcMem | NoAccess | DstReg),
3417 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3420 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3422 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3423 I(SrcImmFAddr | No64, em_call_far), N,
3424 II(ImplicitOps | Stack, em_pushf, pushf),
3425 II(ImplicitOps | Stack, em_popf, popf), N, N,
3427 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3428 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3429 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3430 I2bv(SrcSI | DstDI | String, em_cmp),
3432 I2bv(DstAcc | SrcImm, em_test),
3433 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3434 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3435 I2bv(SrcAcc | DstDI | String, em_cmp),
3437 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3439 X8(I(DstReg | SrcImm | Mov, em_mov)),
3441 D2bv(DstMem | SrcImmByte | ModRM),
3442 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3443 I(ImplicitOps | Stack, em_ret),
3444 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3445 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3446 G(ByteOp, group11), G(0, group11),
3448 N, N, N, I(ImplicitOps | Stack, em_ret_far),
3449 D(ImplicitOps), DI(SrcImmByte, intn),
3450 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3452 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3455 N, N, N, N, N, N, N, N,
3457 X3(I(SrcImmByte, em_loop)),
3458 I(SrcImmByte, em_jcxz),
3459 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3460 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3462 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3463 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3464 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3465 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3467 N, DI(ImplicitOps, icebp), N, N,
3468 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3469 G(ByteOp, group3), G(0, group3),
3471 D(ImplicitOps), D(ImplicitOps),
3472 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3473 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3476 static struct opcode twobyte_table[256] = {
3478 G(0, group6), GD(0, &group7), N, N,
3479 N, I(ImplicitOps | VendorSpecific, em_syscall),
3480 II(ImplicitOps | Priv, em_clts, clts), N,
3481 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3482 N, D(ImplicitOps | ModRM), N, N,
3484 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3486 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3487 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3488 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
3489 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
3491 N, N, N, N, N, N, N, N,
3493 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3494 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3495 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3496 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3497 I(ImplicitOps | VendorSpecific, em_sysenter),
3498 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3500 N, N, N, N, N, N, N, N,
3502 X16(D(DstReg | SrcMem | ModRM | Mov)),
3504 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3509 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3514 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3518 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3520 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3521 DI(ImplicitOps, cpuid), I(DstMem | SrcReg | ModRM | BitOp, em_bt),
3522 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3523 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3525 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3526 DI(ImplicitOps, rsm),
3527 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3528 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3529 D(DstMem | SrcReg | Src2CL | ModRM),
3530 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3532 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3533 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3534 I(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3535 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3536 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3537 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3541 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3542 I(DstReg | SrcMem | ModRM, em_bsf), I(DstReg | SrcMem | ModRM, em_bsr),
3543 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3545 D2bv(DstMem | SrcReg | ModRM | Lock),
3546 N, D(DstMem | SrcReg | ModRM | Mov),
3547 N, N, N, GD(0, &group9),
3548 N, N, N, N, N, N, N, N,
3550 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3552 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3554 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3571 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3575 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3581 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3582 unsigned size, bool sign_extension)
3584 int rc = X86EMUL_CONTINUE;
3588 op->addr.mem.ea = ctxt->_eip;
3589 /* NB. Immediates are sign-extended as necessary. */
3590 switch (op->bytes) {
3592 op->val = insn_fetch(s8, ctxt);
3595 op->val = insn_fetch(s16, ctxt);
3598 op->val = insn_fetch(s32, ctxt);
3601 if (!sign_extension) {
3602 switch (op->bytes) {
3610 op->val &= 0xffffffff;
3618 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
3621 int rc = X86EMUL_CONTINUE;
3625 decode_register_operand(ctxt, op);
3628 rc = decode_imm(ctxt, op, 1, false);
3631 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3635 if ((ctxt->d & BitOp) && op == &ctxt->dst)
3636 fetch_bit_operand(ctxt);
3637 op->orig_val = op->val;
3640 ctxt->memop.bytes = 8;
3644 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3645 op->addr.reg = &ctxt->regs[VCPU_REGS_RAX];
3646 fetch_register_operand(op);
3647 op->orig_val = op->val;
3651 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3653 register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]);
3654 op->addr.mem.seg = VCPU_SREG_ES;
3660 op->addr.reg = &ctxt->regs[VCPU_REGS_RDX];
3661 fetch_register_operand(op);
3665 op->val = ctxt->regs[VCPU_REGS_RCX] & 0xff;
3668 rc = decode_imm(ctxt, op, 1, true);
3675 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
3678 ctxt->memop.bytes = 1;
3681 ctxt->memop.bytes = 2;
3684 ctxt->memop.bytes = 4;
3687 rc = decode_imm(ctxt, op, 2, false);
3690 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
3694 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3696 register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]);
3697 op->addr.mem.seg = seg_override(ctxt);
3702 op->addr.mem.ea = ctxt->_eip;
3703 op->bytes = ctxt->op_bytes + 2;
3704 insn_fetch_arr(op->valptr, op->bytes, ctxt);
3707 ctxt->memop.bytes = ctxt->op_bytes + 2;
3710 op->val = VCPU_SREG_ES;
3713 op->val = VCPU_SREG_CS;
3716 op->val = VCPU_SREG_SS;
3719 op->val = VCPU_SREG_DS;
3722 op->val = VCPU_SREG_FS;
3725 op->val = VCPU_SREG_GS;
3728 /* Special instructions do their own operand decoding. */
3730 op->type = OP_NONE; /* Disable writeback. */
3738 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3740 int rc = X86EMUL_CONTINUE;
3741 int mode = ctxt->mode;
3742 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3743 bool op_prefix = false;
3744 struct opcode opcode;
3746 ctxt->memop.type = OP_NONE;
3747 ctxt->memopp = NULL;
3748 ctxt->_eip = ctxt->eip;
3749 ctxt->fetch.start = ctxt->_eip;
3750 ctxt->fetch.end = ctxt->fetch.start + insn_len;
3752 memcpy(ctxt->fetch.data, insn, insn_len);
3755 case X86EMUL_MODE_REAL:
3756 case X86EMUL_MODE_VM86:
3757 case X86EMUL_MODE_PROT16:
3758 def_op_bytes = def_ad_bytes = 2;
3760 case X86EMUL_MODE_PROT32:
3761 def_op_bytes = def_ad_bytes = 4;
3763 #ifdef CONFIG_X86_64
3764 case X86EMUL_MODE_PROT64:
3770 return EMULATION_FAILED;
3773 ctxt->op_bytes = def_op_bytes;
3774 ctxt->ad_bytes = def_ad_bytes;
3776 /* Legacy prefixes. */
3778 switch (ctxt->b = insn_fetch(u8, ctxt)) {
3779 case 0x66: /* operand-size override */
3781 /* switch between 2/4 bytes */
3782 ctxt->op_bytes = def_op_bytes ^ 6;
3784 case 0x67: /* address-size override */
3785 if (mode == X86EMUL_MODE_PROT64)
3786 /* switch between 4/8 bytes */
3787 ctxt->ad_bytes = def_ad_bytes ^ 12;
3789 /* switch between 2/4 bytes */
3790 ctxt->ad_bytes = def_ad_bytes ^ 6;
3792 case 0x26: /* ES override */
3793 case 0x2e: /* CS override */
3794 case 0x36: /* SS override */
3795 case 0x3e: /* DS override */
3796 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
3798 case 0x64: /* FS override */
3799 case 0x65: /* GS override */
3800 set_seg_override(ctxt, ctxt->b & 7);
3802 case 0x40 ... 0x4f: /* REX */
3803 if (mode != X86EMUL_MODE_PROT64)
3805 ctxt->rex_prefix = ctxt->b;
3807 case 0xf0: /* LOCK */
3808 ctxt->lock_prefix = 1;
3810 case 0xf2: /* REPNE/REPNZ */
3811 case 0xf3: /* REP/REPE/REPZ */
3812 ctxt->rep_prefix = ctxt->b;
3818 /* Any legacy prefix after a REX prefix nullifies its effect. */
3820 ctxt->rex_prefix = 0;
3826 if (ctxt->rex_prefix & 8)
3827 ctxt->op_bytes = 8; /* REX.W */
3829 /* Opcode byte(s). */
3830 opcode = opcode_table[ctxt->b];
3831 /* Two-byte opcode? */
3832 if (ctxt->b == 0x0f) {
3834 ctxt->b = insn_fetch(u8, ctxt);
3835 opcode = twobyte_table[ctxt->b];
3837 ctxt->d = opcode.flags;
3839 while (ctxt->d & GroupMask) {
3840 switch (ctxt->d & GroupMask) {
3842 ctxt->modrm = insn_fetch(u8, ctxt);
3844 goffset = (ctxt->modrm >> 3) & 7;
3845 opcode = opcode.u.group[goffset];
3848 ctxt->modrm = insn_fetch(u8, ctxt);
3850 goffset = (ctxt->modrm >> 3) & 7;
3851 if ((ctxt->modrm >> 6) == 3)
3852 opcode = opcode.u.gdual->mod3[goffset];
3854 opcode = opcode.u.gdual->mod012[goffset];
3857 goffset = ctxt->modrm & 7;
3858 opcode = opcode.u.group[goffset];
3861 if (ctxt->rep_prefix && op_prefix)
3862 return EMULATION_FAILED;
3863 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
3864 switch (simd_prefix) {
3865 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3866 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3867 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3868 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3872 return EMULATION_FAILED;
3875 ctxt->d &= ~(u64)GroupMask;
3876 ctxt->d |= opcode.flags;
3879 ctxt->execute = opcode.u.execute;
3880 ctxt->check_perm = opcode.check_perm;
3881 ctxt->intercept = opcode.intercept;
3884 if (ctxt->d == 0 || (ctxt->d & Undefined))
3885 return EMULATION_FAILED;
3887 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3888 return EMULATION_FAILED;
3890 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
3893 if (ctxt->d & Op3264) {
3894 if (mode == X86EMUL_MODE_PROT64)
3901 ctxt->op_bytes = 16;
3903 /* ModRM and SIB bytes. */
3904 if (ctxt->d & ModRM) {
3905 rc = decode_modrm(ctxt, &ctxt->memop);
3906 if (!ctxt->has_seg_override)
3907 set_seg_override(ctxt, ctxt->modrm_seg);
3908 } else if (ctxt->d & MemAbs)
3909 rc = decode_abs(ctxt, &ctxt->memop);
3910 if (rc != X86EMUL_CONTINUE)
3913 if (!ctxt->has_seg_override)
3914 set_seg_override(ctxt, VCPU_SREG_DS);
3916 ctxt->memop.addr.mem.seg = seg_override(ctxt);
3918 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
3919 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
3922 * Decode and fetch the source operand: register, memory
3925 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
3926 if (rc != X86EMUL_CONTINUE)
3930 * Decode and fetch the second source operand: register, memory
3933 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
3934 if (rc != X86EMUL_CONTINUE)
3937 /* Decode and fetch the destination operand: register or memory. */
3938 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
3941 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
3942 ctxt->memopp->addr.mem.ea += ctxt->_eip;
3944 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
3947 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
3949 return ctxt->d & PageTable;
3952 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3954 /* The second termination condition only applies for REPE
3955 * and REPNE. Test if the repeat string operation prefix is
3956 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3957 * corresponding termination condition according to:
3958 * - if REPE/REPZ and ZF = 0 then done
3959 * - if REPNE/REPNZ and ZF = 1 then done
3961 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
3962 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
3963 && (((ctxt->rep_prefix == REPE_PREFIX) &&
3964 ((ctxt->eflags & EFLG_ZF) == 0))
3965 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
3966 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3972 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3974 struct x86_emulate_ops *ops = ctxt->ops;
3975 int rc = X86EMUL_CONTINUE;
3976 int saved_dst_type = ctxt->dst.type;
3978 ctxt->mem_read.pos = 0;
3980 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
3981 rc = emulate_ud(ctxt);
3985 /* LOCK prefix is allowed only with some instructions */
3986 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
3987 rc = emulate_ud(ctxt);
3991 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
3992 rc = emulate_ud(ctxt);
3997 && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
3998 || !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
3999 rc = emulate_ud(ctxt);
4003 if ((ctxt->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4004 rc = emulate_nm(ctxt);
4008 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4009 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4010 X86_ICPT_PRE_EXCEPT);
4011 if (rc != X86EMUL_CONTINUE)
4015 /* Privileged instruction can be executed only in CPL=0 */
4016 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4017 rc = emulate_gp(ctxt, 0);
4021 /* Instruction can only be executed in protected mode */
4022 if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
4023 rc = emulate_ud(ctxt);
4027 /* Do instruction specific permission checks */
4028 if (ctxt->check_perm) {
4029 rc = ctxt->check_perm(ctxt);
4030 if (rc != X86EMUL_CONTINUE)
4034 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4035 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4036 X86_ICPT_POST_EXCEPT);
4037 if (rc != X86EMUL_CONTINUE)
4041 if (ctxt->rep_prefix && (ctxt->d & String)) {
4042 /* All REP prefixes have the same first termination condition */
4043 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) {
4044 ctxt->eip = ctxt->_eip;
4049 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4050 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4051 ctxt->src.valptr, ctxt->src.bytes);
4052 if (rc != X86EMUL_CONTINUE)
4054 ctxt->src.orig_val64 = ctxt->src.val64;
4057 if (ctxt->src2.type == OP_MEM) {
4058 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4059 &ctxt->src2.val, ctxt->src2.bytes);
4060 if (rc != X86EMUL_CONTINUE)
4064 if ((ctxt->d & DstMask) == ImplicitOps)
4068 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4069 /* optimisation - avoid slow emulated read if Mov */
4070 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4071 &ctxt->dst.val, ctxt->dst.bytes);
4072 if (rc != X86EMUL_CONTINUE)
4075 ctxt->dst.orig_val = ctxt->dst.val;
4079 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4080 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4081 X86_ICPT_POST_MEMACCESS);
4082 if (rc != X86EMUL_CONTINUE)
4086 if (ctxt->execute) {
4087 rc = ctxt->execute(ctxt);
4088 if (rc != X86EMUL_CONTINUE)
4097 case 0x40 ... 0x47: /* inc r16/r32 */
4098 emulate_1op(ctxt, "inc");
4100 case 0x48 ... 0x4f: /* dec r16/r32 */
4101 emulate_1op(ctxt, "dec");
4103 case 0x63: /* movsxd */
4104 if (ctxt->mode != X86EMUL_MODE_PROT64)
4105 goto cannot_emulate;
4106 ctxt->dst.val = (s32) ctxt->src.val;
4108 case 0x70 ... 0x7f: /* jcc (short) */
4109 if (test_cc(ctxt->b, ctxt->eflags))
4110 jmp_rel(ctxt, ctxt->src.val);
4112 case 0x8d: /* lea r16/r32, m */
4113 ctxt->dst.val = ctxt->src.addr.mem.ea;
4115 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4116 if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX])
4120 case 0x98: /* cbw/cwde/cdqe */
4121 switch (ctxt->op_bytes) {
4122 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4123 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4124 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4130 case 0xcc: /* int3 */
4131 rc = emulate_int(ctxt, 3);
4133 case 0xcd: /* int n */
4134 rc = emulate_int(ctxt, ctxt->src.val);
4136 case 0xce: /* into */
4137 if (ctxt->eflags & EFLG_OF)
4138 rc = emulate_int(ctxt, 4);
4140 case 0xd0 ... 0xd1: /* Grp2 */
4143 case 0xd2 ... 0xd3: /* Grp2 */
4144 ctxt->src.val = ctxt->regs[VCPU_REGS_RCX];
4147 case 0xe9: /* jmp rel */
4148 case 0xeb: /* jmp rel short */
4149 jmp_rel(ctxt, ctxt->src.val);
4150 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4152 case 0xf4: /* hlt */
4153 ctxt->ops->halt(ctxt);
4155 case 0xf5: /* cmc */
4156 /* complement carry flag from eflags reg */
4157 ctxt->eflags ^= EFLG_CF;
4159 case 0xf8: /* clc */
4160 ctxt->eflags &= ~EFLG_CF;
4162 case 0xf9: /* stc */
4163 ctxt->eflags |= EFLG_CF;
4165 case 0xfc: /* cld */
4166 ctxt->eflags &= ~EFLG_DF;
4168 case 0xfd: /* std */
4169 ctxt->eflags |= EFLG_DF;
4172 goto cannot_emulate;
4175 if (rc != X86EMUL_CONTINUE)
4179 rc = writeback(ctxt);
4180 if (rc != X86EMUL_CONTINUE)
4184 * restore dst type in case the decoding will be reused
4185 * (happens for string instruction )
4187 ctxt->dst.type = saved_dst_type;
4189 if ((ctxt->d & SrcMask) == SrcSI)
4190 string_addr_inc(ctxt, seg_override(ctxt),
4191 VCPU_REGS_RSI, &ctxt->src);
4193 if ((ctxt->d & DstMask) == DstDI)
4194 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4197 if (ctxt->rep_prefix && (ctxt->d & String)) {
4198 struct read_cache *r = &ctxt->io_read;
4199 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
4201 if (!string_insn_completed(ctxt)) {
4203 * Re-enter guest when pio read ahead buffer is empty
4204 * or, if it is not used, after each 1024 iteration.
4206 if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) &&
4207 (r->end == 0 || r->end != r->pos)) {
4209 * Reset read cache. Usually happens before
4210 * decode, but since instruction is restarted
4211 * we have to do it here.
4213 ctxt->mem_read.end = 0;
4214 return EMULATION_RESTART;
4216 goto done; /* skip rip writeback */
4220 ctxt->eip = ctxt->_eip;
4223 if (rc == X86EMUL_PROPAGATE_FAULT)
4224 ctxt->have_exception = true;
4225 if (rc == X86EMUL_INTERCEPTED)
4226 return EMULATION_INTERCEPTED;
4228 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4232 case 0x09: /* wbinvd */
4233 (ctxt->ops->wbinvd)(ctxt);
4235 case 0x08: /* invd */
4236 case 0x0d: /* GrpP (prefetch) */
4237 case 0x18: /* Grp16 (prefetch/nop) */
4239 case 0x20: /* mov cr, reg */
4240 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4242 case 0x21: /* mov from dr to reg */
4243 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4245 case 0x40 ... 0x4f: /* cmov */
4246 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4247 if (!test_cc(ctxt->b, ctxt->eflags))
4248 ctxt->dst.type = OP_NONE; /* no writeback */
4250 case 0x80 ... 0x8f: /* jnz rel, etc*/
4251 if (test_cc(ctxt->b, ctxt->eflags))
4252 jmp_rel(ctxt, ctxt->src.val);
4254 case 0x90 ... 0x9f: /* setcc r/m8 */
4255 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4257 case 0xa4: /* shld imm8, r, r/m */
4258 case 0xa5: /* shld cl, r, r/m */
4259 emulate_2op_cl(ctxt, "shld");
4261 case 0xac: /* shrd imm8, r, r/m */
4262 case 0xad: /* shrd cl, r, r/m */
4263 emulate_2op_cl(ctxt, "shrd");
4265 case 0xae: /* clflush */
4267 case 0xb6 ... 0xb7: /* movzx */
4268 ctxt->dst.bytes = ctxt->op_bytes;
4269 ctxt->dst.val = (ctxt->d & ByteOp) ? (u8) ctxt->src.val
4270 : (u16) ctxt->src.val;
4272 case 0xbe ... 0xbf: /* movsx */
4273 ctxt->dst.bytes = ctxt->op_bytes;
4274 ctxt->dst.val = (ctxt->d & ByteOp) ? (s8) ctxt->src.val :
4275 (s16) ctxt->src.val;
4277 case 0xc0 ... 0xc1: /* xadd */
4278 emulate_2op_SrcV(ctxt, "add");
4279 /* Write back the register source. */
4280 ctxt->src.val = ctxt->dst.orig_val;
4281 write_register_operand(&ctxt->src);
4283 case 0xc3: /* movnti */
4284 ctxt->dst.bytes = ctxt->op_bytes;
4285 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4286 (u64) ctxt->src.val;
4289 goto cannot_emulate;
4292 if (rc != X86EMUL_CONTINUE)
4298 return EMULATION_FAILED;