1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
32 * Opcode effective-address decode tables.
33 * Note that we only emulate instructions that have at least one memory
34 * operand (excluding implicit stack references). We assume that stack
35 * references and instruction fetches will never occur in special memory
36 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
40 /* Operand sizes: 8-bit operands or specified/overridden size. */
41 #define ByteOp (1<<0) /* 8-bit operands. */
42 /* Destination operand type. */
43 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
44 #define DstReg (2<<1) /* Register operand. */
45 #define DstMem (3<<1) /* Memory operand. */
46 #define DstAcc (4<<1) /* Destination Accumulator */
47 #define DstDI (5<<1) /* Destination is in ES:(E)DI */
48 #define DstMem64 (6<<1) /* 64bit memory operand */
49 #define DstImmUByte (7<<1) /* 8-bit unsigned immediate operand */
50 #define DstMask (7<<1)
51 /* Source operand type. */
52 #define SrcNone (0<<4) /* No source operand. */
53 #define SrcReg (1<<4) /* Register operand. */
54 #define SrcMem (2<<4) /* Memory operand. */
55 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
56 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
57 #define SrcImm (5<<4) /* Immediate operand. */
58 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
59 #define SrcOne (7<<4) /* Implied '1' */
60 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
61 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
62 #define SrcSI (0xa<<4) /* Source is in the DS:RSI */
63 #define SrcImmFAddr (0xb<<4) /* Source is immediate far address */
64 #define SrcMemFAddr (0xc<<4) /* Source is far address in memory */
65 #define SrcAcc (0xd<<4) /* Source Accumulator */
66 #define SrcImmU16 (0xe<<4) /* Immediate operand, unsigned, 16 bits */
67 #define SrcMask (0xf<<4)
68 /* Generic ModRM decode. */
70 /* Destination is only written; never read. */
73 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
74 #define String (1<<12) /* String instruction (rep capable) */
75 #define Stack (1<<13) /* Stack instruction (push/pop) */
76 #define GroupMask (7<<14) /* Opcode uses one of the group mechanisms */
77 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
78 #define GroupDual (2<<14) /* Alternate decoding of mod == 3 */
79 #define Prefix (3<<14) /* Instruction varies with 66/f2/f3 prefix */
80 #define RMExt (4<<14) /* Opcode extension in ModRM r/m if mod == 3 */
81 #define Sse (1<<17) /* SSE Vector instruction */
83 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
84 #define VendorSpecific (1<<22) /* Vendor specific instruction */
85 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
86 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
87 #define Undefined (1<<25) /* No Such Instruction */
88 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
89 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
91 /* Source 2 operand type */
92 #define Src2None (0<<29)
93 #define Src2CL (1<<29)
94 #define Src2ImmByte (2<<29)
95 #define Src2One (3<<29)
96 #define Src2Imm (4<<29)
97 #define Src2Mask (7<<29)
100 #define X3(x...) X2(x), x
101 #define X4(x...) X2(x), X2(x)
102 #define X5(x...) X4(x), x
103 #define X6(x...) X4(x), X2(x)
104 #define X7(x...) X4(x), X3(x)
105 #define X8(x...) X4(x), X4(x)
106 #define X16(x...) X8(x), X8(x)
112 int (*execute)(struct x86_emulate_ctxt *ctxt);
113 struct opcode *group;
114 struct group_dual *gdual;
115 struct gprefix *gprefix;
117 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
121 struct opcode mod012[8];
122 struct opcode mod3[8];
126 struct opcode pfx_no;
127 struct opcode pfx_66;
128 struct opcode pfx_f2;
129 struct opcode pfx_f3;
132 /* EFLAGS bit definitions. */
133 #define EFLG_ID (1<<21)
134 #define EFLG_VIP (1<<20)
135 #define EFLG_VIF (1<<19)
136 #define EFLG_AC (1<<18)
137 #define EFLG_VM (1<<17)
138 #define EFLG_RF (1<<16)
139 #define EFLG_IOPL (3<<12)
140 #define EFLG_NT (1<<14)
141 #define EFLG_OF (1<<11)
142 #define EFLG_DF (1<<10)
143 #define EFLG_IF (1<<9)
144 #define EFLG_TF (1<<8)
145 #define EFLG_SF (1<<7)
146 #define EFLG_ZF (1<<6)
147 #define EFLG_AF (1<<4)
148 #define EFLG_PF (1<<2)
149 #define EFLG_CF (1<<0)
151 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
152 #define EFLG_RESERVED_ONE_MASK 2
155 * Instruction emulation:
156 * Most instructions are emulated directly via a fragment of inline assembly
157 * code. This allows us to save/restore EFLAGS and thus very easily pick up
158 * any modified flags.
161 #if defined(CONFIG_X86_64)
162 #define _LO32 "k" /* force 32-bit operand */
163 #define _STK "%%rsp" /* stack pointer */
164 #elif defined(__i386__)
165 #define _LO32 "" /* force 32-bit operand */
166 #define _STK "%%esp" /* stack pointer */
170 * These EFLAGS bits are restored from saved value during emulation, and
171 * any changes are written back to the saved value after emulation.
173 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
175 /* Before executing instruction: restore necessary bits in EFLAGS. */
176 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
177 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
178 "movl %"_sav",%"_LO32 _tmp"; " \
181 "movl %"_msk",%"_LO32 _tmp"; " \
182 "andl %"_LO32 _tmp",("_STK"); " \
184 "notl %"_LO32 _tmp"; " \
185 "andl %"_LO32 _tmp",("_STK"); " \
186 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
188 "orl %"_LO32 _tmp",("_STK"); " \
192 /* After executing instruction: write-back necessary bits in EFLAGS. */
193 #define _POST_EFLAGS(_sav, _msk, _tmp) \
194 /* _sav |= EFLAGS & _msk; */ \
197 "andl %"_msk",%"_LO32 _tmp"; " \
198 "orl %"_LO32 _tmp",%"_sav"; "
206 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix, _dsttype) \
208 __asm__ __volatile__ ( \
209 _PRE_EFLAGS("0", "4", "2") \
210 _op _suffix " %"_x"3,%1; " \
211 _POST_EFLAGS("0", "4", "2") \
212 : "=m" (_eflags), "+q" (*(_dsttype*)&(_dst).val),\
214 : _y ((_src).val), "i" (EFLAGS_MASK)); \
218 /* Raw emulation: instruction has two explicit operands. */
219 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
221 unsigned long _tmp; \
223 switch ((_dst).bytes) { \
225 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w",u16);\
228 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l",u32);\
231 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q",u64)); \
236 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
238 unsigned long _tmp; \
239 switch ((_dst).bytes) { \
241 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b",u8); \
244 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
245 _wx, _wy, _lx, _ly, _qx, _qy); \
250 /* Source operand is byte-sized and may be restricted to just %cl. */
251 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
252 __emulate_2op(_op, _src, _dst, _eflags, \
253 "b", "c", "b", "c", "b", "c", "b", "c")
255 /* Source operand is byte, word, long or quad sized. */
256 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
257 __emulate_2op(_op, _src, _dst, _eflags, \
258 "b", "q", "w", "r", _LO32, "r", "", "r")
260 /* Source operand is word, long or quad sized. */
261 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
262 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
263 "w", "r", _LO32, "r", "", "r")
265 /* Instruction has three operands and one operand is stored in ECX register */
266 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
268 unsigned long _tmp; \
269 _type _clv = (_cl).val; \
270 _type _srcv = (_src).val; \
271 _type _dstv = (_dst).val; \
273 __asm__ __volatile__ ( \
274 _PRE_EFLAGS("0", "5", "2") \
275 _op _suffix " %4,%1 \n" \
276 _POST_EFLAGS("0", "5", "2") \
277 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
278 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
281 (_cl).val = (unsigned long) _clv; \
282 (_src).val = (unsigned long) _srcv; \
283 (_dst).val = (unsigned long) _dstv; \
286 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
288 switch ((_dst).bytes) { \
290 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
291 "w", unsigned short); \
294 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
295 "l", unsigned int); \
298 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
299 "q", unsigned long)); \
304 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
306 unsigned long _tmp; \
308 __asm__ __volatile__ ( \
309 _PRE_EFLAGS("0", "3", "2") \
310 _op _suffix " %1; " \
311 _POST_EFLAGS("0", "3", "2") \
312 : "=m" (_eflags), "+m" ((_dst).val), \
314 : "i" (EFLAGS_MASK)); \
317 /* Instruction has only one explicit operand (no source operand). */
318 #define emulate_1op(_op, _dst, _eflags) \
320 switch ((_dst).bytes) { \
321 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
322 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
323 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
324 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
328 #define __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, _suffix) \
330 unsigned long _tmp; \
332 __asm__ __volatile__ ( \
333 _PRE_EFLAGS("0", "4", "1") \
334 _op _suffix " %5; " \
335 _POST_EFLAGS("0", "4", "1") \
336 : "=m" (_eflags), "=&r" (_tmp), \
337 "+a" (_rax), "+d" (_rdx) \
338 : "i" (EFLAGS_MASK), "m" ((_src).val), \
339 "a" (_rax), "d" (_rdx)); \
342 #define __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _suffix, _ex) \
344 unsigned long _tmp; \
346 __asm__ __volatile__ ( \
347 _PRE_EFLAGS("0", "5", "1") \
349 _op _suffix " %6; " \
351 _POST_EFLAGS("0", "5", "1") \
352 ".pushsection .fixup,\"ax\" \n\t" \
353 "3: movb $1, %4 \n\t" \
356 _ASM_EXTABLE(1b, 3b) \
357 : "=m" (_eflags), "=&r" (_tmp), \
358 "+a" (_rax), "+d" (_rdx), "+qm"(_ex) \
359 : "i" (EFLAGS_MASK), "m" ((_src).val), \
360 "a" (_rax), "d" (_rdx)); \
363 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
364 #define emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags) \
366 switch((_src).bytes) { \
368 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
372 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
376 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
380 ON64(__emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
386 #define emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _ex) \
388 switch((_src).bytes) { \
390 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
391 _eflags, "b", _ex); \
394 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
395 _eflags, "w", _ex); \
398 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
399 _eflags, "l", _ex); \
402 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
403 _eflags, "q", _ex)); \
408 /* Fetch next part of the instruction being emulated. */
409 #define insn_fetch(_type, _size, _eip) \
410 ({ unsigned long _x; \
411 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
412 if (rc != X86EMUL_CONTINUE) \
418 #define insn_fetch_arr(_arr, _size, _eip) \
419 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
420 if (rc != X86EMUL_CONTINUE) \
425 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
426 enum x86_intercept intercept,
427 enum x86_intercept_stage stage)
429 struct x86_instruction_info info = {
430 .intercept = intercept,
431 .rep_prefix = ctxt->decode.rep_prefix,
432 .modrm_mod = ctxt->decode.modrm_mod,
433 .modrm_reg = ctxt->decode.modrm_reg,
434 .modrm_rm = ctxt->decode.modrm_rm,
435 .src_val = ctxt->decode.src.val64,
436 .src_bytes = ctxt->decode.src.bytes,
437 .dst_bytes = ctxt->decode.dst.bytes,
438 .ad_bytes = ctxt->decode.ad_bytes,
439 .next_rip = ctxt->eip,
442 return ctxt->ops->intercept(ctxt, &info, stage);
445 static inline unsigned long ad_mask(struct decode_cache *c)
447 return (1UL << (c->ad_bytes << 3)) - 1;
450 /* Access/update address held in a register, based on addressing mode. */
451 static inline unsigned long
452 address_mask(struct decode_cache *c, unsigned long reg)
454 if (c->ad_bytes == sizeof(unsigned long))
457 return reg & ad_mask(c);
460 static inline unsigned long
461 register_address(struct decode_cache *c, unsigned long reg)
463 return address_mask(c, reg);
467 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
469 if (c->ad_bytes == sizeof(unsigned long))
472 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
475 static inline void jmp_rel(struct decode_cache *c, int rel)
477 register_address_increment(c, &c->eip, rel);
480 static u32 desc_limit_scaled(struct desc_struct *desc)
482 u32 limit = get_desc_limit(desc);
484 return desc->g ? (limit << 12) | 0xfff : limit;
487 static void set_seg_override(struct decode_cache *c, int seg)
489 c->has_seg_override = true;
490 c->seg_override = seg;
493 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
494 struct x86_emulate_ops *ops, int seg)
496 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
499 return ops->get_cached_segment_base(ctxt, seg);
502 static unsigned seg_override(struct x86_emulate_ctxt *ctxt,
503 struct decode_cache *c)
505 if (!c->has_seg_override)
508 return c->seg_override;
511 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
512 u32 error, bool valid)
514 ctxt->exception.vector = vec;
515 ctxt->exception.error_code = error;
516 ctxt->exception.error_code_valid = valid;
517 return X86EMUL_PROPAGATE_FAULT;
520 static int emulate_db(struct x86_emulate_ctxt *ctxt)
522 return emulate_exception(ctxt, DB_VECTOR, 0, false);
525 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
527 return emulate_exception(ctxt, GP_VECTOR, err, true);
530 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
532 return emulate_exception(ctxt, SS_VECTOR, err, true);
535 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
537 return emulate_exception(ctxt, UD_VECTOR, 0, false);
540 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
542 return emulate_exception(ctxt, TS_VECTOR, err, true);
545 static int emulate_de(struct x86_emulate_ctxt *ctxt)
547 return emulate_exception(ctxt, DE_VECTOR, 0, false);
550 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
552 return emulate_exception(ctxt, NM_VECTOR, 0, false);
555 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
558 struct desc_struct desc;
560 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
564 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
569 struct desc_struct desc;
571 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
572 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
575 static int __linearize(struct x86_emulate_ctxt *ctxt,
576 struct segmented_address addr,
577 unsigned size, bool write, bool fetch,
580 struct decode_cache *c = &ctxt->decode;
581 struct desc_struct desc;
588 la = seg_base(ctxt, ctxt->ops, addr.seg) + addr.ea;
589 switch (ctxt->mode) {
590 case X86EMUL_MODE_REAL:
592 case X86EMUL_MODE_PROT64:
593 if (((signed long)la << 16) >> 16 != la)
594 return emulate_gp(ctxt, 0);
597 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
601 /* code segment or read-only data segment */
602 if (((desc.type & 8) || !(desc.type & 2)) && write)
604 /* unreadable code segment */
605 if (!fetch && (desc.type & 8) && !(desc.type & 2))
607 lim = desc_limit_scaled(&desc);
608 if ((desc.type & 8) || !(desc.type & 4)) {
609 /* expand-up segment */
610 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
613 /* exapand-down segment */
614 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
616 lim = desc.d ? 0xffffffff : 0xffff;
617 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
620 cpl = ctxt->ops->cpl(ctxt);
623 if (!(desc.type & 8)) {
627 } else if ((desc.type & 8) && !(desc.type & 4)) {
628 /* nonconforming code segment */
631 } else if ((desc.type & 8) && (desc.type & 4)) {
632 /* conforming code segment */
638 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : c->ad_bytes != 8)
641 return X86EMUL_CONTINUE;
643 if (addr.seg == VCPU_SREG_SS)
644 return emulate_ss(ctxt, addr.seg);
646 return emulate_gp(ctxt, addr.seg);
649 static int linearize(struct x86_emulate_ctxt *ctxt,
650 struct segmented_address addr,
651 unsigned size, bool write,
654 return __linearize(ctxt, addr, size, write, false, linear);
658 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
659 struct segmented_address addr,
666 rc = linearize(ctxt, addr, size, false, &linear);
667 if (rc != X86EMUL_CONTINUE)
669 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
672 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
673 struct x86_emulate_ops *ops,
674 unsigned long eip, u8 *dest)
676 struct fetch_cache *fc = &ctxt->decode.fetch;
680 if (eip == fc->end) {
681 unsigned long linear;
682 struct segmented_address addr = { .seg=VCPU_SREG_CS, .ea=eip};
683 cur_size = fc->end - fc->start;
684 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
685 rc = __linearize(ctxt, addr, size, false, true, &linear);
686 if (rc != X86EMUL_CONTINUE)
688 rc = ops->fetch(ctxt, linear, fc->data + cur_size,
689 size, &ctxt->exception);
690 if (rc != X86EMUL_CONTINUE)
694 *dest = fc->data[eip - fc->start];
695 return X86EMUL_CONTINUE;
698 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
699 struct x86_emulate_ops *ops,
700 unsigned long eip, void *dest, unsigned size)
704 /* x86 instructions are limited to 15 bytes. */
705 if (eip + size - ctxt->eip > 15)
706 return X86EMUL_UNHANDLEABLE;
708 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
709 if (rc != X86EMUL_CONTINUE)
712 return X86EMUL_CONTINUE;
716 * Given the 'reg' portion of a ModRM byte, and a register block, return a
717 * pointer into the block that addresses the relevant register.
718 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
720 static void *decode_register(u8 modrm_reg, unsigned long *regs,
725 p = ®s[modrm_reg];
726 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
727 p = (unsigned char *)®s[modrm_reg & 3] + 1;
731 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
732 struct segmented_address addr,
733 u16 *size, unsigned long *address, int op_bytes)
740 rc = segmented_read_std(ctxt, addr, size, 2);
741 if (rc != X86EMUL_CONTINUE)
744 rc = segmented_read_std(ctxt, addr, address, op_bytes);
748 static int test_cc(unsigned int condition, unsigned int flags)
752 switch ((condition & 15) >> 1) {
754 rc |= (flags & EFLG_OF);
756 case 1: /* b/c/nae */
757 rc |= (flags & EFLG_CF);
760 rc |= (flags & EFLG_ZF);
763 rc |= (flags & (EFLG_CF|EFLG_ZF));
766 rc |= (flags & EFLG_SF);
769 rc |= (flags & EFLG_PF);
772 rc |= (flags & EFLG_ZF);
775 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
779 /* Odd condition identifiers (lsb == 1) have inverted sense. */
780 return (!!rc ^ (condition & 1));
783 static void fetch_register_operand(struct operand *op)
787 op->val = *(u8 *)op->addr.reg;
790 op->val = *(u16 *)op->addr.reg;
793 op->val = *(u32 *)op->addr.reg;
796 op->val = *(u64 *)op->addr.reg;
801 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
803 ctxt->ops->get_fpu(ctxt);
805 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
806 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
807 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
808 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
809 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
810 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
811 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
812 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
814 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
815 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
816 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
817 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
818 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
819 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
820 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
821 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
825 ctxt->ops->put_fpu(ctxt);
828 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
831 ctxt->ops->get_fpu(ctxt);
833 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
834 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
835 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
836 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
837 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
838 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
839 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
840 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
842 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
843 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
844 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
845 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
846 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
847 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
848 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
849 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
853 ctxt->ops->put_fpu(ctxt);
856 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
858 struct decode_cache *c,
861 unsigned reg = c->modrm_reg;
862 int highbyte_regs = c->rex_prefix == 0;
865 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
871 read_sse_reg(ctxt, &op->vec_val, reg);
876 if ((c->d & ByteOp) && !inhibit_bytereg) {
877 op->addr.reg = decode_register(reg, c->regs, highbyte_regs);
880 op->addr.reg = decode_register(reg, c->regs, 0);
881 op->bytes = c->op_bytes;
883 fetch_register_operand(op);
884 op->orig_val = op->val;
887 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
888 struct x86_emulate_ops *ops,
891 struct decode_cache *c = &ctxt->decode;
893 int index_reg = 0, base_reg = 0, scale;
894 int rc = X86EMUL_CONTINUE;
898 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
899 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
900 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
903 c->modrm = insn_fetch(u8, 1, c->eip);
904 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
905 c->modrm_reg |= (c->modrm & 0x38) >> 3;
906 c->modrm_rm |= (c->modrm & 0x07);
907 c->modrm_seg = VCPU_SREG_DS;
909 if (c->modrm_mod == 3) {
911 op->bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
912 op->addr.reg = decode_register(c->modrm_rm,
913 c->regs, c->d & ByteOp);
917 op->addr.xmm = c->modrm_rm;
918 read_sse_reg(ctxt, &op->vec_val, c->modrm_rm);
921 fetch_register_operand(op);
927 if (c->ad_bytes == 2) {
928 unsigned bx = c->regs[VCPU_REGS_RBX];
929 unsigned bp = c->regs[VCPU_REGS_RBP];
930 unsigned si = c->regs[VCPU_REGS_RSI];
931 unsigned di = c->regs[VCPU_REGS_RDI];
933 /* 16-bit ModR/M decode. */
934 switch (c->modrm_mod) {
936 if (c->modrm_rm == 6)
937 modrm_ea += insn_fetch(u16, 2, c->eip);
940 modrm_ea += insn_fetch(s8, 1, c->eip);
943 modrm_ea += insn_fetch(u16, 2, c->eip);
946 switch (c->modrm_rm) {
966 if (c->modrm_mod != 0)
973 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
974 (c->modrm_rm == 6 && c->modrm_mod != 0))
975 c->modrm_seg = VCPU_SREG_SS;
976 modrm_ea = (u16)modrm_ea;
978 /* 32/64-bit ModR/M decode. */
979 if ((c->modrm_rm & 7) == 4) {
980 sib = insn_fetch(u8, 1, c->eip);
981 index_reg |= (sib >> 3) & 7;
985 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
986 modrm_ea += insn_fetch(s32, 4, c->eip);
988 modrm_ea += c->regs[base_reg];
990 modrm_ea += c->regs[index_reg] << scale;
991 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
992 if (ctxt->mode == X86EMUL_MODE_PROT64)
995 modrm_ea += c->regs[c->modrm_rm];
996 switch (c->modrm_mod) {
998 if (c->modrm_rm == 5)
999 modrm_ea += insn_fetch(s32, 4, c->eip);
1002 modrm_ea += insn_fetch(s8, 1, c->eip);
1005 modrm_ea += insn_fetch(s32, 4, c->eip);
1009 op->addr.mem.ea = modrm_ea;
1014 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1015 struct x86_emulate_ops *ops,
1018 struct decode_cache *c = &ctxt->decode;
1019 int rc = X86EMUL_CONTINUE;
1022 switch (c->ad_bytes) {
1024 op->addr.mem.ea = insn_fetch(u16, 2, c->eip);
1027 op->addr.mem.ea = insn_fetch(u32, 4, c->eip);
1030 op->addr.mem.ea = insn_fetch(u64, 8, c->eip);
1037 static void fetch_bit_operand(struct decode_cache *c)
1041 if (c->dst.type == OP_MEM && c->src.type == OP_REG) {
1042 mask = ~(c->dst.bytes * 8 - 1);
1044 if (c->src.bytes == 2)
1045 sv = (s16)c->src.val & (s16)mask;
1046 else if (c->src.bytes == 4)
1047 sv = (s32)c->src.val & (s32)mask;
1049 c->dst.addr.mem.ea += (sv >> 3);
1052 /* only subword offset */
1053 c->src.val &= (c->dst.bytes << 3) - 1;
1056 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1057 struct x86_emulate_ops *ops,
1058 unsigned long addr, void *dest, unsigned size)
1061 struct read_cache *mc = &ctxt->decode.mem_read;
1064 int n = min(size, 8u);
1066 if (mc->pos < mc->end)
1069 rc = ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1071 if (rc != X86EMUL_CONTINUE)
1076 memcpy(dest, mc->data + mc->pos, n);
1081 return X86EMUL_CONTINUE;
1084 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1085 struct segmented_address addr,
1092 rc = linearize(ctxt, addr, size, false, &linear);
1093 if (rc != X86EMUL_CONTINUE)
1095 return read_emulated(ctxt, ctxt->ops, linear, data, size);
1098 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1099 struct segmented_address addr,
1106 rc = linearize(ctxt, addr, size, true, &linear);
1107 if (rc != X86EMUL_CONTINUE)
1109 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1113 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1114 struct segmented_address addr,
1115 const void *orig_data, const void *data,
1121 rc = linearize(ctxt, addr, size, true, &linear);
1122 if (rc != X86EMUL_CONTINUE)
1124 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1125 size, &ctxt->exception);
1128 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1129 struct x86_emulate_ops *ops,
1130 unsigned int size, unsigned short port,
1133 struct read_cache *rc = &ctxt->decode.io_read;
1135 if (rc->pos == rc->end) { /* refill pio read ahead */
1136 struct decode_cache *c = &ctxt->decode;
1137 unsigned int in_page, n;
1138 unsigned int count = c->rep_prefix ?
1139 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
1140 in_page = (ctxt->eflags & EFLG_DF) ?
1141 offset_in_page(c->regs[VCPU_REGS_RDI]) :
1142 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
1143 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1147 rc->pos = rc->end = 0;
1148 if (!ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1153 memcpy(dest, rc->data + rc->pos, size);
1158 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1159 struct x86_emulate_ops *ops,
1160 u16 selector, struct desc_ptr *dt)
1162 if (selector & 1 << 2) {
1163 struct desc_struct desc;
1166 memset (dt, 0, sizeof *dt);
1167 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1170 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1171 dt->address = get_desc_base(&desc);
1173 ops->get_gdt(ctxt, dt);
1176 /* allowed just for 8 bytes segments */
1177 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1178 struct x86_emulate_ops *ops,
1179 u16 selector, struct desc_struct *desc)
1182 u16 index = selector >> 3;
1186 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1188 if (dt.size < index * 8 + 7)
1189 return emulate_gp(ctxt, selector & 0xfffc);
1190 addr = dt.address + index * 8;
1191 ret = ops->read_std(ctxt, addr, desc, sizeof *desc, &ctxt->exception);
1196 /* allowed just for 8 bytes segments */
1197 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1198 struct x86_emulate_ops *ops,
1199 u16 selector, struct desc_struct *desc)
1202 u16 index = selector >> 3;
1206 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1208 if (dt.size < index * 8 + 7)
1209 return emulate_gp(ctxt, selector & 0xfffc);
1211 addr = dt.address + index * 8;
1212 ret = ops->write_std(ctxt, addr, desc, sizeof *desc, &ctxt->exception);
1217 /* Does not support long mode */
1218 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1219 struct x86_emulate_ops *ops,
1220 u16 selector, int seg)
1222 struct desc_struct seg_desc;
1224 unsigned err_vec = GP_VECTOR;
1226 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1229 memset(&seg_desc, 0, sizeof seg_desc);
1231 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1232 || ctxt->mode == X86EMUL_MODE_REAL) {
1233 /* set real mode segment descriptor */
1234 set_desc_base(&seg_desc, selector << 4);
1235 set_desc_limit(&seg_desc, 0xffff);
1242 /* NULL selector is not valid for TR, CS and SS */
1243 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1247 /* TR should be in GDT only */
1248 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1251 if (null_selector) /* for NULL selector skip all following checks */
1254 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1255 if (ret != X86EMUL_CONTINUE)
1258 err_code = selector & 0xfffc;
1259 err_vec = GP_VECTOR;
1261 /* can't load system descriptor into segment selecor */
1262 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1266 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1272 cpl = ops->cpl(ctxt);
1277 * segment is not a writable data segment or segment
1278 * selector's RPL != CPL or segment selector's RPL != CPL
1280 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1284 if (!(seg_desc.type & 8))
1287 if (seg_desc.type & 4) {
1293 if (rpl > cpl || dpl != cpl)
1296 /* CS(RPL) <- CPL */
1297 selector = (selector & 0xfffc) | cpl;
1300 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1303 case VCPU_SREG_LDTR:
1304 if (seg_desc.s || seg_desc.type != 2)
1307 default: /* DS, ES, FS, or GS */
1309 * segment is not a data or readable code segment or
1310 * ((segment is a data or nonconforming code segment)
1311 * and (both RPL and CPL > DPL))
1313 if ((seg_desc.type & 0xa) == 0x8 ||
1314 (((seg_desc.type & 0xc) != 0xc) &&
1315 (rpl > dpl && cpl > dpl)))
1321 /* mark segment as accessed */
1323 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1324 if (ret != X86EMUL_CONTINUE)
1328 ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1329 return X86EMUL_CONTINUE;
1331 emulate_exception(ctxt, err_vec, err_code, true);
1332 return X86EMUL_PROPAGATE_FAULT;
1335 static void write_register_operand(struct operand *op)
1337 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1338 switch (op->bytes) {
1340 *(u8 *)op->addr.reg = (u8)op->val;
1343 *(u16 *)op->addr.reg = (u16)op->val;
1346 *op->addr.reg = (u32)op->val;
1347 break; /* 64b: zero-extend */
1349 *op->addr.reg = op->val;
1354 static int writeback(struct x86_emulate_ctxt *ctxt)
1357 struct decode_cache *c = &ctxt->decode;
1359 switch (c->dst.type) {
1361 write_register_operand(&c->dst);
1365 rc = segmented_cmpxchg(ctxt,
1371 rc = segmented_write(ctxt,
1375 if (rc != X86EMUL_CONTINUE)
1379 write_sse_reg(ctxt, &c->dst.vec_val, c->dst.addr.xmm);
1387 return X86EMUL_CONTINUE;
1390 static int em_push(struct x86_emulate_ctxt *ctxt)
1392 struct decode_cache *c = &ctxt->decode;
1393 struct segmented_address addr;
1395 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1396 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1397 addr.seg = VCPU_SREG_SS;
1399 /* Disable writeback. */
1400 c->dst.type = OP_NONE;
1401 return segmented_write(ctxt, addr, &c->src.val, c->op_bytes);
1404 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1405 void *dest, int len)
1407 struct decode_cache *c = &ctxt->decode;
1409 struct segmented_address addr;
1411 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1412 addr.seg = VCPU_SREG_SS;
1413 rc = segmented_read(ctxt, addr, dest, len);
1414 if (rc != X86EMUL_CONTINUE)
1417 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1421 static int em_pop(struct x86_emulate_ctxt *ctxt)
1423 struct decode_cache *c = &ctxt->decode;
1425 return emulate_pop(ctxt, &c->dst.val, c->op_bytes);
1428 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1429 struct x86_emulate_ops *ops,
1430 void *dest, int len)
1433 unsigned long val, change_mask;
1434 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1435 int cpl = ops->cpl(ctxt);
1437 rc = emulate_pop(ctxt, &val, len);
1438 if (rc != X86EMUL_CONTINUE)
1441 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1442 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1444 switch(ctxt->mode) {
1445 case X86EMUL_MODE_PROT64:
1446 case X86EMUL_MODE_PROT32:
1447 case X86EMUL_MODE_PROT16:
1449 change_mask |= EFLG_IOPL;
1451 change_mask |= EFLG_IF;
1453 case X86EMUL_MODE_VM86:
1455 return emulate_gp(ctxt, 0);
1456 change_mask |= EFLG_IF;
1458 default: /* real mode */
1459 change_mask |= (EFLG_IOPL | EFLG_IF);
1463 *(unsigned long *)dest =
1464 (ctxt->eflags & ~change_mask) | (val & change_mask);
1469 static int em_popf(struct x86_emulate_ctxt *ctxt)
1471 struct decode_cache *c = &ctxt->decode;
1473 c->dst.type = OP_REG;
1474 c->dst.addr.reg = &ctxt->eflags;
1475 c->dst.bytes = c->op_bytes;
1476 return emulate_popf(ctxt, ctxt->ops, &c->dst.val, c->op_bytes);
1479 static int emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1480 struct x86_emulate_ops *ops, int seg)
1482 struct decode_cache *c = &ctxt->decode;
1484 c->src.val = get_segment_selector(ctxt, seg);
1486 return em_push(ctxt);
1489 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1490 struct x86_emulate_ops *ops, int seg)
1492 struct decode_cache *c = &ctxt->decode;
1493 unsigned long selector;
1496 rc = emulate_pop(ctxt, &selector, c->op_bytes);
1497 if (rc != X86EMUL_CONTINUE)
1500 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1504 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1506 struct decode_cache *c = &ctxt->decode;
1507 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1508 int rc = X86EMUL_CONTINUE;
1509 int reg = VCPU_REGS_RAX;
1511 while (reg <= VCPU_REGS_RDI) {
1512 (reg == VCPU_REGS_RSP) ?
1513 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1516 if (rc != X86EMUL_CONTINUE)
1525 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1527 struct decode_cache *c = &ctxt->decode;
1529 c->src.val = (unsigned long)ctxt->eflags;
1530 return em_push(ctxt);
1533 static int em_popa(struct x86_emulate_ctxt *ctxt)
1535 struct decode_cache *c = &ctxt->decode;
1536 int rc = X86EMUL_CONTINUE;
1537 int reg = VCPU_REGS_RDI;
1539 while (reg >= VCPU_REGS_RAX) {
1540 if (reg == VCPU_REGS_RSP) {
1541 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1546 rc = emulate_pop(ctxt, &c->regs[reg], c->op_bytes);
1547 if (rc != X86EMUL_CONTINUE)
1554 int emulate_int_real(struct x86_emulate_ctxt *ctxt,
1555 struct x86_emulate_ops *ops, int irq)
1557 struct decode_cache *c = &ctxt->decode;
1564 /* TODO: Add limit checks */
1565 c->src.val = ctxt->eflags;
1567 if (rc != X86EMUL_CONTINUE)
1570 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1572 c->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1574 if (rc != X86EMUL_CONTINUE)
1577 c->src.val = c->eip;
1579 if (rc != X86EMUL_CONTINUE)
1582 ops->get_idt(ctxt, &dt);
1584 eip_addr = dt.address + (irq << 2);
1585 cs_addr = dt.address + (irq << 2) + 2;
1587 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1588 if (rc != X86EMUL_CONTINUE)
1591 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1592 if (rc != X86EMUL_CONTINUE)
1595 rc = load_segment_descriptor(ctxt, ops, cs, VCPU_SREG_CS);
1596 if (rc != X86EMUL_CONTINUE)
1604 static int emulate_int(struct x86_emulate_ctxt *ctxt,
1605 struct x86_emulate_ops *ops, int irq)
1607 switch(ctxt->mode) {
1608 case X86EMUL_MODE_REAL:
1609 return emulate_int_real(ctxt, ops, irq);
1610 case X86EMUL_MODE_VM86:
1611 case X86EMUL_MODE_PROT16:
1612 case X86EMUL_MODE_PROT32:
1613 case X86EMUL_MODE_PROT64:
1615 /* Protected mode interrupts unimplemented yet */
1616 return X86EMUL_UNHANDLEABLE;
1620 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
1621 struct x86_emulate_ops *ops)
1623 struct decode_cache *c = &ctxt->decode;
1624 int rc = X86EMUL_CONTINUE;
1625 unsigned long temp_eip = 0;
1626 unsigned long temp_eflags = 0;
1627 unsigned long cs = 0;
1628 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1629 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1630 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1631 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1633 /* TODO: Add stack limit check */
1635 rc = emulate_pop(ctxt, &temp_eip, c->op_bytes);
1637 if (rc != X86EMUL_CONTINUE)
1640 if (temp_eip & ~0xffff)
1641 return emulate_gp(ctxt, 0);
1643 rc = emulate_pop(ctxt, &cs, c->op_bytes);
1645 if (rc != X86EMUL_CONTINUE)
1648 rc = emulate_pop(ctxt, &temp_eflags, c->op_bytes);
1650 if (rc != X86EMUL_CONTINUE)
1653 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1655 if (rc != X86EMUL_CONTINUE)
1661 if (c->op_bytes == 4)
1662 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1663 else if (c->op_bytes == 2) {
1664 ctxt->eflags &= ~0xffff;
1665 ctxt->eflags |= temp_eflags;
1668 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1669 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1674 static inline int emulate_iret(struct x86_emulate_ctxt *ctxt,
1675 struct x86_emulate_ops* ops)
1677 switch(ctxt->mode) {
1678 case X86EMUL_MODE_REAL:
1679 return emulate_iret_real(ctxt, ops);
1680 case X86EMUL_MODE_VM86:
1681 case X86EMUL_MODE_PROT16:
1682 case X86EMUL_MODE_PROT32:
1683 case X86EMUL_MODE_PROT64:
1685 /* iret from protected mode unimplemented yet */
1686 return X86EMUL_UNHANDLEABLE;
1690 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt)
1692 struct decode_cache *c = &ctxt->decode;
1694 return emulate_pop(ctxt, &c->dst.val, c->dst.bytes);
1697 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1699 struct decode_cache *c = &ctxt->decode;
1700 switch (c->modrm_reg) {
1702 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1705 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1708 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1711 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1713 case 4: /* sal/shl */
1714 case 6: /* sal/shl */
1715 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1718 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1721 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1726 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1727 struct x86_emulate_ops *ops)
1729 struct decode_cache *c = &ctxt->decode;
1730 unsigned long *rax = &c->regs[VCPU_REGS_RAX];
1731 unsigned long *rdx = &c->regs[VCPU_REGS_RDX];
1734 switch (c->modrm_reg) {
1735 case 0 ... 1: /* test */
1736 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1739 c->dst.val = ~c->dst.val;
1742 emulate_1op("neg", c->dst, ctxt->eflags);
1745 emulate_1op_rax_rdx("mul", c->src, *rax, *rdx, ctxt->eflags);
1748 emulate_1op_rax_rdx("imul", c->src, *rax, *rdx, ctxt->eflags);
1751 emulate_1op_rax_rdx_ex("div", c->src, *rax, *rdx,
1755 emulate_1op_rax_rdx_ex("idiv", c->src, *rax, *rdx,
1759 return X86EMUL_UNHANDLEABLE;
1762 return emulate_de(ctxt);
1763 return X86EMUL_CONTINUE;
1766 static int emulate_grp45(struct x86_emulate_ctxt *ctxt)
1768 struct decode_cache *c = &ctxt->decode;
1769 int rc = X86EMUL_CONTINUE;
1771 switch (c->modrm_reg) {
1773 emulate_1op("inc", c->dst, ctxt->eflags);
1776 emulate_1op("dec", c->dst, ctxt->eflags);
1778 case 2: /* call near abs */ {
1781 c->eip = c->src.val;
1782 c->src.val = old_eip;
1786 case 4: /* jmp abs */
1787 c->eip = c->src.val;
1796 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1797 struct x86_emulate_ops *ops)
1799 struct decode_cache *c = &ctxt->decode;
1800 u64 old = c->dst.orig_val64;
1802 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1803 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1804 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1805 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1806 ctxt->eflags &= ~EFLG_ZF;
1808 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1809 (u32) c->regs[VCPU_REGS_RBX];
1811 ctxt->eflags |= EFLG_ZF;
1813 return X86EMUL_CONTINUE;
1816 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1817 struct x86_emulate_ops *ops)
1819 struct decode_cache *c = &ctxt->decode;
1823 rc = emulate_pop(ctxt, &c->eip, c->op_bytes);
1824 if (rc != X86EMUL_CONTINUE)
1826 if (c->op_bytes == 4)
1827 c->eip = (u32)c->eip;
1828 rc = emulate_pop(ctxt, &cs, c->op_bytes);
1829 if (rc != X86EMUL_CONTINUE)
1831 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1835 static int emulate_load_segment(struct x86_emulate_ctxt *ctxt,
1836 struct x86_emulate_ops *ops, int seg)
1838 struct decode_cache *c = &ctxt->decode;
1842 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
1844 rc = load_segment_descriptor(ctxt, ops, sel, seg);
1845 if (rc != X86EMUL_CONTINUE)
1848 c->dst.val = c->src.val;
1853 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1854 struct x86_emulate_ops *ops, struct desc_struct *cs,
1855 struct desc_struct *ss)
1859 memset(cs, 0, sizeof(struct desc_struct));
1860 ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1861 memset(ss, 0, sizeof(struct desc_struct));
1863 cs->l = 0; /* will be adjusted later */
1864 set_desc_base(cs, 0); /* flat segment */
1865 cs->g = 1; /* 4kb granularity */
1866 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1867 cs->type = 0x0b; /* Read, Execute, Accessed */
1869 cs->dpl = 0; /* will be adjusted later */
1873 set_desc_base(ss, 0); /* flat segment */
1874 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1875 ss->g = 1; /* 4kb granularity */
1877 ss->type = 0x03; /* Read/Write, Accessed */
1878 ss->d = 1; /* 32bit stack segment */
1884 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1886 struct decode_cache *c = &ctxt->decode;
1887 struct desc_struct cs, ss;
1892 /* syscall is not available in real mode */
1893 if (ctxt->mode == X86EMUL_MODE_REAL ||
1894 ctxt->mode == X86EMUL_MODE_VM86)
1895 return emulate_ud(ctxt);
1897 ops->get_msr(ctxt, MSR_EFER, &efer);
1898 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1900 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1902 cs_sel = (u16)(msr_data & 0xfffc);
1903 ss_sel = (u16)(msr_data + 8);
1905 if (efer & EFER_LMA) {
1909 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1910 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1912 c->regs[VCPU_REGS_RCX] = c->eip;
1913 if (efer & EFER_LMA) {
1914 #ifdef CONFIG_X86_64
1915 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1918 ctxt->mode == X86EMUL_MODE_PROT64 ?
1919 MSR_LSTAR : MSR_CSTAR, &msr_data);
1922 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1923 ctxt->eflags &= ~(msr_data | EFLG_RF);
1927 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1928 c->eip = (u32)msr_data;
1930 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1933 return X86EMUL_CONTINUE;
1937 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1939 struct decode_cache *c = &ctxt->decode;
1940 struct desc_struct cs, ss;
1945 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1946 /* inject #GP if in real mode */
1947 if (ctxt->mode == X86EMUL_MODE_REAL)
1948 return emulate_gp(ctxt, 0);
1950 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1951 * Therefore, we inject an #UD.
1953 if (ctxt->mode == X86EMUL_MODE_PROT64)
1954 return emulate_ud(ctxt);
1956 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1958 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1959 switch (ctxt->mode) {
1960 case X86EMUL_MODE_PROT32:
1961 if ((msr_data & 0xfffc) == 0x0)
1962 return emulate_gp(ctxt, 0);
1964 case X86EMUL_MODE_PROT64:
1965 if (msr_data == 0x0)
1966 return emulate_gp(ctxt, 0);
1970 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1971 cs_sel = (u16)msr_data;
1972 cs_sel &= ~SELECTOR_RPL_MASK;
1973 ss_sel = cs_sel + 8;
1974 ss_sel &= ~SELECTOR_RPL_MASK;
1975 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
1980 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1981 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1983 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
1986 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
1987 c->regs[VCPU_REGS_RSP] = msr_data;
1989 return X86EMUL_CONTINUE;
1993 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1995 struct decode_cache *c = &ctxt->decode;
1996 struct desc_struct cs, ss;
2001 /* inject #GP if in real mode or Virtual 8086 mode */
2002 if (ctxt->mode == X86EMUL_MODE_REAL ||
2003 ctxt->mode == X86EMUL_MODE_VM86)
2004 return emulate_gp(ctxt, 0);
2006 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2008 if ((c->rex_prefix & 0x8) != 0x0)
2009 usermode = X86EMUL_MODE_PROT64;
2011 usermode = X86EMUL_MODE_PROT32;
2015 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2017 case X86EMUL_MODE_PROT32:
2018 cs_sel = (u16)(msr_data + 16);
2019 if ((msr_data & 0xfffc) == 0x0)
2020 return emulate_gp(ctxt, 0);
2021 ss_sel = (u16)(msr_data + 24);
2023 case X86EMUL_MODE_PROT64:
2024 cs_sel = (u16)(msr_data + 32);
2025 if (msr_data == 0x0)
2026 return emulate_gp(ctxt, 0);
2027 ss_sel = cs_sel + 8;
2032 cs_sel |= SELECTOR_RPL_MASK;
2033 ss_sel |= SELECTOR_RPL_MASK;
2035 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2036 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2038 c->eip = c->regs[VCPU_REGS_RDX];
2039 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
2041 return X86EMUL_CONTINUE;
2044 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
2045 struct x86_emulate_ops *ops)
2048 if (ctxt->mode == X86EMUL_MODE_REAL)
2050 if (ctxt->mode == X86EMUL_MODE_VM86)
2052 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2053 return ops->cpl(ctxt) > iopl;
2056 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2057 struct x86_emulate_ops *ops,
2060 struct desc_struct tr_seg;
2063 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2064 unsigned mask = (1 << len) - 1;
2067 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2070 if (desc_limit_scaled(&tr_seg) < 103)
2072 base = get_desc_base(&tr_seg);
2073 #ifdef CONFIG_X86_64
2074 base |= ((u64)base3) << 32;
2076 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2077 if (r != X86EMUL_CONTINUE)
2079 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2081 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2082 if (r != X86EMUL_CONTINUE)
2084 if ((perm >> bit_idx) & mask)
2089 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2090 struct x86_emulate_ops *ops,
2096 if (emulator_bad_iopl(ctxt, ops))
2097 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
2100 ctxt->perm_ok = true;
2105 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2106 struct x86_emulate_ops *ops,
2107 struct tss_segment_16 *tss)
2109 struct decode_cache *c = &ctxt->decode;
2112 tss->flag = ctxt->eflags;
2113 tss->ax = c->regs[VCPU_REGS_RAX];
2114 tss->cx = c->regs[VCPU_REGS_RCX];
2115 tss->dx = c->regs[VCPU_REGS_RDX];
2116 tss->bx = c->regs[VCPU_REGS_RBX];
2117 tss->sp = c->regs[VCPU_REGS_RSP];
2118 tss->bp = c->regs[VCPU_REGS_RBP];
2119 tss->si = c->regs[VCPU_REGS_RSI];
2120 tss->di = c->regs[VCPU_REGS_RDI];
2122 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2123 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2124 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2125 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2126 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2129 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2130 struct x86_emulate_ops *ops,
2131 struct tss_segment_16 *tss)
2133 struct decode_cache *c = &ctxt->decode;
2137 ctxt->eflags = tss->flag | 2;
2138 c->regs[VCPU_REGS_RAX] = tss->ax;
2139 c->regs[VCPU_REGS_RCX] = tss->cx;
2140 c->regs[VCPU_REGS_RDX] = tss->dx;
2141 c->regs[VCPU_REGS_RBX] = tss->bx;
2142 c->regs[VCPU_REGS_RSP] = tss->sp;
2143 c->regs[VCPU_REGS_RBP] = tss->bp;
2144 c->regs[VCPU_REGS_RSI] = tss->si;
2145 c->regs[VCPU_REGS_RDI] = tss->di;
2148 * SDM says that segment selectors are loaded before segment
2151 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2152 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2153 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2154 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2155 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2158 * Now load segment descriptors. If fault happenes at this stage
2159 * it is handled in a context of new task
2161 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
2162 if (ret != X86EMUL_CONTINUE)
2164 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2165 if (ret != X86EMUL_CONTINUE)
2167 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2168 if (ret != X86EMUL_CONTINUE)
2170 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2171 if (ret != X86EMUL_CONTINUE)
2173 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2174 if (ret != X86EMUL_CONTINUE)
2177 return X86EMUL_CONTINUE;
2180 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2181 struct x86_emulate_ops *ops,
2182 u16 tss_selector, u16 old_tss_sel,
2183 ulong old_tss_base, struct desc_struct *new_desc)
2185 struct tss_segment_16 tss_seg;
2187 u32 new_tss_base = get_desc_base(new_desc);
2189 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2191 if (ret != X86EMUL_CONTINUE)
2192 /* FIXME: need to provide precise fault address */
2195 save_state_to_tss16(ctxt, ops, &tss_seg);
2197 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2199 if (ret != X86EMUL_CONTINUE)
2200 /* FIXME: need to provide precise fault address */
2203 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2205 if (ret != X86EMUL_CONTINUE)
2206 /* FIXME: need to provide precise fault address */
2209 if (old_tss_sel != 0xffff) {
2210 tss_seg.prev_task_link = old_tss_sel;
2212 ret = ops->write_std(ctxt, new_tss_base,
2213 &tss_seg.prev_task_link,
2214 sizeof tss_seg.prev_task_link,
2216 if (ret != X86EMUL_CONTINUE)
2217 /* FIXME: need to provide precise fault address */
2221 return load_state_from_tss16(ctxt, ops, &tss_seg);
2224 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2225 struct x86_emulate_ops *ops,
2226 struct tss_segment_32 *tss)
2228 struct decode_cache *c = &ctxt->decode;
2230 tss->cr3 = ops->get_cr(ctxt, 3);
2232 tss->eflags = ctxt->eflags;
2233 tss->eax = c->regs[VCPU_REGS_RAX];
2234 tss->ecx = c->regs[VCPU_REGS_RCX];
2235 tss->edx = c->regs[VCPU_REGS_RDX];
2236 tss->ebx = c->regs[VCPU_REGS_RBX];
2237 tss->esp = c->regs[VCPU_REGS_RSP];
2238 tss->ebp = c->regs[VCPU_REGS_RBP];
2239 tss->esi = c->regs[VCPU_REGS_RSI];
2240 tss->edi = c->regs[VCPU_REGS_RDI];
2242 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2243 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2244 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2245 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2246 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2247 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2248 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2251 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2252 struct x86_emulate_ops *ops,
2253 struct tss_segment_32 *tss)
2255 struct decode_cache *c = &ctxt->decode;
2258 if (ops->set_cr(ctxt, 3, tss->cr3))
2259 return emulate_gp(ctxt, 0);
2261 ctxt->eflags = tss->eflags | 2;
2262 c->regs[VCPU_REGS_RAX] = tss->eax;
2263 c->regs[VCPU_REGS_RCX] = tss->ecx;
2264 c->regs[VCPU_REGS_RDX] = tss->edx;
2265 c->regs[VCPU_REGS_RBX] = tss->ebx;
2266 c->regs[VCPU_REGS_RSP] = tss->esp;
2267 c->regs[VCPU_REGS_RBP] = tss->ebp;
2268 c->regs[VCPU_REGS_RSI] = tss->esi;
2269 c->regs[VCPU_REGS_RDI] = tss->edi;
2272 * SDM says that segment selectors are loaded before segment
2275 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2276 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2277 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2278 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2279 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2280 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2281 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2284 * Now load segment descriptors. If fault happenes at this stage
2285 * it is handled in a context of new task
2287 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2288 if (ret != X86EMUL_CONTINUE)
2290 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2291 if (ret != X86EMUL_CONTINUE)
2293 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2294 if (ret != X86EMUL_CONTINUE)
2296 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2297 if (ret != X86EMUL_CONTINUE)
2299 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2300 if (ret != X86EMUL_CONTINUE)
2302 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2303 if (ret != X86EMUL_CONTINUE)
2305 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2306 if (ret != X86EMUL_CONTINUE)
2309 return X86EMUL_CONTINUE;
2312 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2313 struct x86_emulate_ops *ops,
2314 u16 tss_selector, u16 old_tss_sel,
2315 ulong old_tss_base, struct desc_struct *new_desc)
2317 struct tss_segment_32 tss_seg;
2319 u32 new_tss_base = get_desc_base(new_desc);
2321 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2323 if (ret != X86EMUL_CONTINUE)
2324 /* FIXME: need to provide precise fault address */
2327 save_state_to_tss32(ctxt, ops, &tss_seg);
2329 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2331 if (ret != X86EMUL_CONTINUE)
2332 /* FIXME: need to provide precise fault address */
2335 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2337 if (ret != X86EMUL_CONTINUE)
2338 /* FIXME: need to provide precise fault address */
2341 if (old_tss_sel != 0xffff) {
2342 tss_seg.prev_task_link = old_tss_sel;
2344 ret = ops->write_std(ctxt, new_tss_base,
2345 &tss_seg.prev_task_link,
2346 sizeof tss_seg.prev_task_link,
2348 if (ret != X86EMUL_CONTINUE)
2349 /* FIXME: need to provide precise fault address */
2353 return load_state_from_tss32(ctxt, ops, &tss_seg);
2356 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2357 struct x86_emulate_ops *ops,
2358 u16 tss_selector, int reason,
2359 bool has_error_code, u32 error_code)
2361 struct desc_struct curr_tss_desc, next_tss_desc;
2363 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2364 ulong old_tss_base =
2365 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2368 /* FIXME: old_tss_base == ~0 ? */
2370 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2371 if (ret != X86EMUL_CONTINUE)
2373 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2374 if (ret != X86EMUL_CONTINUE)
2377 /* FIXME: check that next_tss_desc is tss */
2379 if (reason != TASK_SWITCH_IRET) {
2380 if ((tss_selector & 3) > next_tss_desc.dpl ||
2381 ops->cpl(ctxt) > next_tss_desc.dpl)
2382 return emulate_gp(ctxt, 0);
2385 desc_limit = desc_limit_scaled(&next_tss_desc);
2386 if (!next_tss_desc.p ||
2387 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2388 desc_limit < 0x2b)) {
2389 emulate_ts(ctxt, tss_selector & 0xfffc);
2390 return X86EMUL_PROPAGATE_FAULT;
2393 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2394 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2395 write_segment_descriptor(ctxt, ops, old_tss_sel,
2399 if (reason == TASK_SWITCH_IRET)
2400 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2402 /* set back link to prev task only if NT bit is set in eflags
2403 note that old_tss_sel is not used afetr this point */
2404 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2405 old_tss_sel = 0xffff;
2407 if (next_tss_desc.type & 8)
2408 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2409 old_tss_base, &next_tss_desc);
2411 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2412 old_tss_base, &next_tss_desc);
2413 if (ret != X86EMUL_CONTINUE)
2416 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2417 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2419 if (reason != TASK_SWITCH_IRET) {
2420 next_tss_desc.type |= (1 << 1); /* set busy flag */
2421 write_segment_descriptor(ctxt, ops, tss_selector,
2425 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2426 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2428 if (has_error_code) {
2429 struct decode_cache *c = &ctxt->decode;
2431 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2433 c->src.val = (unsigned long) error_code;
2434 ret = em_push(ctxt);
2440 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2441 u16 tss_selector, int reason,
2442 bool has_error_code, u32 error_code)
2444 struct x86_emulate_ops *ops = ctxt->ops;
2445 struct decode_cache *c = &ctxt->decode;
2449 c->dst.type = OP_NONE;
2451 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2452 has_error_code, error_code);
2454 if (rc == X86EMUL_CONTINUE)
2457 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2460 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2461 int reg, struct operand *op)
2463 struct decode_cache *c = &ctxt->decode;
2464 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2466 register_address_increment(c, &c->regs[reg], df * op->bytes);
2467 op->addr.mem.ea = register_address(c, c->regs[reg]);
2468 op->addr.mem.seg = seg;
2471 static int em_das(struct x86_emulate_ctxt *ctxt)
2473 struct decode_cache *c = &ctxt->decode;
2475 bool af, cf, old_cf;
2477 cf = ctxt->eflags & X86_EFLAGS_CF;
2483 af = ctxt->eflags & X86_EFLAGS_AF;
2484 if ((al & 0x0f) > 9 || af) {
2486 cf = old_cf | (al >= 250);
2491 if (old_al > 0x99 || old_cf) {
2497 /* Set PF, ZF, SF */
2498 c->src.type = OP_IMM;
2501 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2502 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2504 ctxt->eflags |= X86_EFLAGS_CF;
2506 ctxt->eflags |= X86_EFLAGS_AF;
2507 return X86EMUL_CONTINUE;
2510 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2512 struct decode_cache *c = &ctxt->decode;
2517 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2520 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
2521 if (load_segment_descriptor(ctxt, ctxt->ops, sel, VCPU_SREG_CS))
2522 return X86EMUL_CONTINUE;
2525 memcpy(&c->eip, c->src.valptr, c->op_bytes);
2527 c->src.val = old_cs;
2529 if (rc != X86EMUL_CONTINUE)
2532 c->src.val = old_eip;
2533 return em_push(ctxt);
2536 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2538 struct decode_cache *c = &ctxt->decode;
2541 c->dst.type = OP_REG;
2542 c->dst.addr.reg = &c->eip;
2543 c->dst.bytes = c->op_bytes;
2544 rc = emulate_pop(ctxt, &c->dst.val, c->op_bytes);
2545 if (rc != X86EMUL_CONTINUE)
2547 register_address_increment(c, &c->regs[VCPU_REGS_RSP], c->src.val);
2548 return X86EMUL_CONTINUE;
2551 static int em_add(struct x86_emulate_ctxt *ctxt)
2553 struct decode_cache *c = &ctxt->decode;
2555 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
2556 return X86EMUL_CONTINUE;
2559 static int em_or(struct x86_emulate_ctxt *ctxt)
2561 struct decode_cache *c = &ctxt->decode;
2563 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2564 return X86EMUL_CONTINUE;
2567 static int em_adc(struct x86_emulate_ctxt *ctxt)
2569 struct decode_cache *c = &ctxt->decode;
2571 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
2572 return X86EMUL_CONTINUE;
2575 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2577 struct decode_cache *c = &ctxt->decode;
2579 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
2580 return X86EMUL_CONTINUE;
2583 static int em_and(struct x86_emulate_ctxt *ctxt)
2585 struct decode_cache *c = &ctxt->decode;
2587 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
2588 return X86EMUL_CONTINUE;
2591 static int em_sub(struct x86_emulate_ctxt *ctxt)
2593 struct decode_cache *c = &ctxt->decode;
2595 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
2596 return X86EMUL_CONTINUE;
2599 static int em_xor(struct x86_emulate_ctxt *ctxt)
2601 struct decode_cache *c = &ctxt->decode;
2603 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
2604 return X86EMUL_CONTINUE;
2607 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2609 struct decode_cache *c = &ctxt->decode;
2611 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2612 /* Disable writeback. */
2613 c->dst.type = OP_NONE;
2614 return X86EMUL_CONTINUE;
2617 static int em_imul(struct x86_emulate_ctxt *ctxt)
2619 struct decode_cache *c = &ctxt->decode;
2621 emulate_2op_SrcV_nobyte("imul", c->src, c->dst, ctxt->eflags);
2622 return X86EMUL_CONTINUE;
2625 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2627 struct decode_cache *c = &ctxt->decode;
2629 c->dst.val = c->src2.val;
2630 return em_imul(ctxt);
2633 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2635 struct decode_cache *c = &ctxt->decode;
2637 c->dst.type = OP_REG;
2638 c->dst.bytes = c->src.bytes;
2639 c->dst.addr.reg = &c->regs[VCPU_REGS_RDX];
2640 c->dst.val = ~((c->src.val >> (c->src.bytes * 8 - 1)) - 1);
2642 return X86EMUL_CONTINUE;
2645 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2647 struct decode_cache *c = &ctxt->decode;
2650 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2651 c->regs[VCPU_REGS_RAX] = (u32)tsc;
2652 c->regs[VCPU_REGS_RDX] = tsc >> 32;
2653 return X86EMUL_CONTINUE;
2656 static int em_mov(struct x86_emulate_ctxt *ctxt)
2658 struct decode_cache *c = &ctxt->decode;
2659 c->dst.val = c->src.val;
2660 return X86EMUL_CONTINUE;
2663 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2665 struct decode_cache *c = &ctxt->decode;
2666 memcpy(&c->dst.vec_val, &c->src.vec_val, c->op_bytes);
2667 return X86EMUL_CONTINUE;
2670 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2672 struct decode_cache *c = &ctxt->decode;
2676 rc = linearize(ctxt, c->src.addr.mem, 1, false, &linear);
2677 if (rc == X86EMUL_CONTINUE)
2678 ctxt->ops->invlpg(ctxt, linear);
2679 /* Disable writeback. */
2680 c->dst.type = OP_NONE;
2681 return X86EMUL_CONTINUE;
2684 static int em_clts(struct x86_emulate_ctxt *ctxt)
2688 cr0 = ctxt->ops->get_cr(ctxt, 0);
2690 ctxt->ops->set_cr(ctxt, 0, cr0);
2691 return X86EMUL_CONTINUE;
2694 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
2696 struct decode_cache *c = &ctxt->decode;
2699 if (c->modrm_mod != 3 || c->modrm_rm != 1)
2700 return X86EMUL_UNHANDLEABLE;
2702 rc = ctxt->ops->fix_hypercall(ctxt);
2703 if (rc != X86EMUL_CONTINUE)
2706 /* Let the processor re-execute the fixed hypercall */
2708 /* Disable writeback. */
2709 c->dst.type = OP_NONE;
2710 return X86EMUL_CONTINUE;
2713 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
2715 struct decode_cache *c = &ctxt->decode;
2716 struct desc_ptr desc_ptr;
2719 rc = read_descriptor(ctxt, c->src.addr.mem,
2720 &desc_ptr.size, &desc_ptr.address,
2722 if (rc != X86EMUL_CONTINUE)
2724 ctxt->ops->set_gdt(ctxt, &desc_ptr);
2725 /* Disable writeback. */
2726 c->dst.type = OP_NONE;
2727 return X86EMUL_CONTINUE;
2730 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2732 struct decode_cache *c = &ctxt->decode;
2735 rc = ctxt->ops->fix_hypercall(ctxt);
2737 /* Disable writeback. */
2738 c->dst.type = OP_NONE;
2742 static int em_lidt(struct x86_emulate_ctxt *ctxt)
2744 struct decode_cache *c = &ctxt->decode;
2745 struct desc_ptr desc_ptr;
2748 rc = read_descriptor(ctxt, c->src.addr.mem,
2749 &desc_ptr.size, &desc_ptr.address,
2751 if (rc != X86EMUL_CONTINUE)
2753 ctxt->ops->set_idt(ctxt, &desc_ptr);
2754 /* Disable writeback. */
2755 c->dst.type = OP_NONE;
2756 return X86EMUL_CONTINUE;
2759 static int em_smsw(struct x86_emulate_ctxt *ctxt)
2761 struct decode_cache *c = &ctxt->decode;
2764 c->dst.val = ctxt->ops->get_cr(ctxt, 0);
2765 return X86EMUL_CONTINUE;
2768 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
2770 struct decode_cache *c = &ctxt->decode;
2771 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2772 | (c->src.val & 0x0f));
2773 c->dst.type = OP_NONE;
2774 return X86EMUL_CONTINUE;
2777 static bool valid_cr(int nr)
2789 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2791 struct decode_cache *c = &ctxt->decode;
2793 if (!valid_cr(c->modrm_reg))
2794 return emulate_ud(ctxt);
2796 return X86EMUL_CONTINUE;
2799 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2801 struct decode_cache *c = &ctxt->decode;
2802 u64 new_val = c->src.val64;
2803 int cr = c->modrm_reg;
2806 static u64 cr_reserved_bits[] = {
2807 0xffffffff00000000ULL,
2808 0, 0, 0, /* CR3 checked later */
2815 return emulate_ud(ctxt);
2817 if (new_val & cr_reserved_bits[cr])
2818 return emulate_gp(ctxt, 0);
2823 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2824 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2825 return emulate_gp(ctxt, 0);
2827 cr4 = ctxt->ops->get_cr(ctxt, 4);
2828 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2830 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2831 !(cr4 & X86_CR4_PAE))
2832 return emulate_gp(ctxt, 0);
2839 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2840 if (efer & EFER_LMA)
2841 rsvd = CR3_L_MODE_RESERVED_BITS;
2842 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
2843 rsvd = CR3_PAE_RESERVED_BITS;
2844 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
2845 rsvd = CR3_NONPAE_RESERVED_BITS;
2848 return emulate_gp(ctxt, 0);
2855 cr4 = ctxt->ops->get_cr(ctxt, 4);
2856 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2858 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2859 return emulate_gp(ctxt, 0);
2865 return X86EMUL_CONTINUE;
2868 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2872 ctxt->ops->get_dr(ctxt, 7, &dr7);
2874 /* Check if DR7.Global_Enable is set */
2875 return dr7 & (1 << 13);
2878 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2880 struct decode_cache *c = &ctxt->decode;
2881 int dr = c->modrm_reg;
2885 return emulate_ud(ctxt);
2887 cr4 = ctxt->ops->get_cr(ctxt, 4);
2888 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2889 return emulate_ud(ctxt);
2891 if (check_dr7_gd(ctxt))
2892 return emulate_db(ctxt);
2894 return X86EMUL_CONTINUE;
2897 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2899 struct decode_cache *c = &ctxt->decode;
2900 u64 new_val = c->src.val64;
2901 int dr = c->modrm_reg;
2903 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2904 return emulate_gp(ctxt, 0);
2906 return check_dr_read(ctxt);
2909 static int check_svme(struct x86_emulate_ctxt *ctxt)
2913 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2915 if (!(efer & EFER_SVME))
2916 return emulate_ud(ctxt);
2918 return X86EMUL_CONTINUE;
2921 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
2923 u64 rax = ctxt->decode.regs[VCPU_REGS_RAX];
2925 /* Valid physical address? */
2926 if (rax & 0xffff000000000000ULL)
2927 return emulate_gp(ctxt, 0);
2929 return check_svme(ctxt);
2932 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
2934 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2936 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
2937 return emulate_ud(ctxt);
2939 return X86EMUL_CONTINUE;
2942 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
2944 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2945 u64 rcx = ctxt->decode.regs[VCPU_REGS_RCX];
2947 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
2949 return emulate_gp(ctxt, 0);
2951 return X86EMUL_CONTINUE;
2954 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
2956 struct decode_cache *c = &ctxt->decode;
2958 c->dst.bytes = min(c->dst.bytes, 4u);
2959 if (!emulator_io_permited(ctxt, ctxt->ops, c->src.val, c->dst.bytes))
2960 return emulate_gp(ctxt, 0);
2962 return X86EMUL_CONTINUE;
2965 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
2967 struct decode_cache *c = &ctxt->decode;
2969 c->src.bytes = min(c->src.bytes, 4u);
2970 if (!emulator_io_permited(ctxt, ctxt->ops, c->dst.val, c->src.bytes))
2971 return emulate_gp(ctxt, 0);
2973 return X86EMUL_CONTINUE;
2976 #define D(_y) { .flags = (_y) }
2977 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2978 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
2979 .check_perm = (_p) }
2981 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
2982 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2983 #define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
2984 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2985 #define II(_f, _e, _i) \
2986 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2987 #define IIP(_f, _e, _i, _p) \
2988 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
2989 .check_perm = (_p) }
2990 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
2992 #define D2bv(_f) D((_f) | ByteOp), D(_f)
2993 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
2994 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
2996 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
2997 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
2998 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3000 static struct opcode group7_rm1[] = {
3001 DI(SrcNone | ModRM | Priv, monitor),
3002 DI(SrcNone | ModRM | Priv, mwait),
3006 static struct opcode group7_rm3[] = {
3007 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
3008 II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
3009 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
3010 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
3011 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
3012 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
3013 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
3014 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
3017 static struct opcode group7_rm7[] = {
3019 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
3023 static struct opcode group1[] = {
3034 static struct opcode group1A[] = {
3035 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
3038 static struct opcode group3[] = {
3039 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
3040 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3041 X4(D(SrcMem | ModRM)),
3044 static struct opcode group4[] = {
3045 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
3049 static struct opcode group5[] = {
3050 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3051 D(SrcMem | ModRM | Stack),
3052 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3053 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
3054 D(SrcMem | ModRM | Stack), N,
3057 static struct opcode group6[] = {
3058 DI(ModRM | Prot, sldt),
3059 DI(ModRM | Prot, str),
3060 DI(ModRM | Prot | Priv, lldt),
3061 DI(ModRM | Prot | Priv, ltr),
3065 static struct group_dual group7 = { {
3066 DI(ModRM | Mov | DstMem | Priv, sgdt),
3067 DI(ModRM | Mov | DstMem | Priv, sidt),
3068 II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
3069 II(ModRM | SrcMem | Priv, em_lidt, lidt),
3070 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3071 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
3072 II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3074 I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
3076 N, EXT(0, group7_rm3),
3077 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3078 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3081 static struct opcode group8[] = {
3083 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
3084 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
3087 static struct group_dual group9 = { {
3088 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
3090 N, N, N, N, N, N, N, N,
3093 static struct opcode group11[] = {
3094 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
3097 static struct gprefix pfx_0f_6f_0f_7f = {
3098 N, N, N, I(Sse, em_movdqu),
3101 static struct opcode opcode_table[256] = {
3103 I6ALU(Lock, em_add),
3104 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3107 D(ImplicitOps | Stack | No64), N,
3109 I6ALU(Lock, em_adc),
3110 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3112 I6ALU(Lock, em_sbb),
3113 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3115 I6ALU(Lock, em_and), N, N,
3117 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3119 I6ALU(Lock, em_xor), N, N,
3121 I6ALU(0, em_cmp), N, N,
3125 X8(I(SrcReg | Stack, em_push)),
3127 X8(I(DstReg | Stack, em_pop)),
3129 I(ImplicitOps | Stack | No64, em_pusha),
3130 I(ImplicitOps | Stack | No64, em_popa),
3131 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3134 I(SrcImm | Mov | Stack, em_push),
3135 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3136 I(SrcImmByte | Mov | Stack, em_push),
3137 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3138 D2bvIP(DstDI | Mov | String, ins, check_perm_in), /* insb, insw/insd */
3139 D2bvIP(SrcSI | ImplicitOps | String, outs, check_perm_out), /* outsb, outsw/outsd */
3143 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
3144 G(DstMem | SrcImm | ModRM | Group, group1),
3145 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
3146 G(DstMem | SrcImmByte | ModRM | Group, group1),
3147 D2bv(DstMem | SrcReg | ModRM), D2bv(DstMem | SrcReg | ModRM | Lock),
3149 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
3150 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3151 D(DstMem | SrcNone | ModRM | Mov), D(ModRM | SrcMem | NoAccess | DstReg),
3152 D(ImplicitOps | SrcMem16 | ModRM), G(0, group1A),
3154 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3156 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3157 I(SrcImmFAddr | No64, em_call_far), N,
3158 II(ImplicitOps | Stack, em_pushf, pushf),
3159 II(ImplicitOps | Stack, em_popf, popf), N, N,
3161 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3162 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
3163 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3164 I2bv(SrcSI | DstDI | String, em_cmp),
3166 D2bv(DstAcc | SrcImm),
3167 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3168 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3169 I2bv(SrcAcc | DstDI | String, em_cmp),
3171 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3173 X8(I(DstReg | SrcImm | Mov, em_mov)),
3175 D2bv(DstMem | SrcImmByte | ModRM),
3176 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3177 D(ImplicitOps | Stack),
3178 D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
3179 G(ByteOp, group11), G(0, group11),
3181 N, N, N, D(ImplicitOps | Stack),
3182 D(ImplicitOps), DI(SrcImmByte, intn),
3183 D(ImplicitOps | No64), DI(ImplicitOps, iret),
3185 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3188 N, N, N, N, N, N, N, N,
3191 D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
3192 D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
3194 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
3195 D(SrcImmFAddr | No64), D(SrcImmByte | ImplicitOps),
3196 D2bvIP(SrcNone | DstAcc, in, check_perm_in),
3197 D2bvIP(SrcAcc | ImplicitOps, out, check_perm_out),
3199 N, DI(ImplicitOps, icebp), N, N,
3200 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3201 G(ByteOp, group3), G(0, group3),
3203 D(ImplicitOps), D(ImplicitOps), D(ImplicitOps), D(ImplicitOps),
3204 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3207 static struct opcode twobyte_table[256] = {
3209 G(0, group6), GD(0, &group7), N, N,
3210 N, D(ImplicitOps | VendorSpecific), DI(ImplicitOps | Priv, clts), N,
3211 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3212 N, D(ImplicitOps | ModRM), N, N,
3214 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3216 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3217 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3218 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3219 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3221 N, N, N, N, N, N, N, N,
3223 DI(ImplicitOps | Priv, wrmsr),
3224 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3225 DI(ImplicitOps | Priv, rdmsr),
3226 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3227 D(ImplicitOps | VendorSpecific), D(ImplicitOps | Priv | VendorSpecific),
3229 N, N, N, N, N, N, N, N,
3231 X16(D(DstReg | SrcMem | ModRM | Mov)),
3233 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3238 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3243 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3247 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3249 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3250 DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3251 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3252 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3254 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3255 DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3256 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3257 D(DstMem | SrcReg | Src2CL | ModRM),
3258 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3260 D2bv(DstMem | SrcReg | ModRM | Lock),
3261 D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3262 D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
3263 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3266 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3267 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3268 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3270 D2bv(DstMem | SrcReg | ModRM | Lock),
3271 N, D(DstMem | SrcReg | ModRM | Mov),
3272 N, N, N, GD(0, &group9),
3273 N, N, N, N, N, N, N, N,
3275 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3277 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3279 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3295 static unsigned imm_size(struct decode_cache *c)
3299 size = (c->d & ByteOp) ? 1 : c->op_bytes;
3305 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3306 unsigned size, bool sign_extension)
3308 struct decode_cache *c = &ctxt->decode;
3309 struct x86_emulate_ops *ops = ctxt->ops;
3310 int rc = X86EMUL_CONTINUE;
3314 op->addr.mem.ea = c->eip;
3315 /* NB. Immediates are sign-extended as necessary. */
3316 switch (op->bytes) {
3318 op->val = insn_fetch(s8, 1, c->eip);
3321 op->val = insn_fetch(s16, 2, c->eip);
3324 op->val = insn_fetch(s32, 4, c->eip);
3327 if (!sign_extension) {
3328 switch (op->bytes) {
3336 op->val &= 0xffffffff;
3345 x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3347 struct x86_emulate_ops *ops = ctxt->ops;
3348 struct decode_cache *c = &ctxt->decode;
3349 int rc = X86EMUL_CONTINUE;
3350 int mode = ctxt->mode;
3351 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3352 bool op_prefix = false;
3353 struct opcode opcode;
3354 struct operand memop = { .type = OP_NONE };
3357 c->fetch.start = c->eip;
3358 c->fetch.end = c->fetch.start + insn_len;
3360 memcpy(c->fetch.data, insn, insn_len);
3363 case X86EMUL_MODE_REAL:
3364 case X86EMUL_MODE_VM86:
3365 case X86EMUL_MODE_PROT16:
3366 def_op_bytes = def_ad_bytes = 2;
3368 case X86EMUL_MODE_PROT32:
3369 def_op_bytes = def_ad_bytes = 4;
3371 #ifdef CONFIG_X86_64
3372 case X86EMUL_MODE_PROT64:
3381 c->op_bytes = def_op_bytes;
3382 c->ad_bytes = def_ad_bytes;
3384 /* Legacy prefixes. */
3386 switch (c->b = insn_fetch(u8, 1, c->eip)) {
3387 case 0x66: /* operand-size override */
3389 /* switch between 2/4 bytes */
3390 c->op_bytes = def_op_bytes ^ 6;
3392 case 0x67: /* address-size override */
3393 if (mode == X86EMUL_MODE_PROT64)
3394 /* switch between 4/8 bytes */
3395 c->ad_bytes = def_ad_bytes ^ 12;
3397 /* switch between 2/4 bytes */
3398 c->ad_bytes = def_ad_bytes ^ 6;
3400 case 0x26: /* ES override */
3401 case 0x2e: /* CS override */
3402 case 0x36: /* SS override */
3403 case 0x3e: /* DS override */
3404 set_seg_override(c, (c->b >> 3) & 3);
3406 case 0x64: /* FS override */
3407 case 0x65: /* GS override */
3408 set_seg_override(c, c->b & 7);
3410 case 0x40 ... 0x4f: /* REX */
3411 if (mode != X86EMUL_MODE_PROT64)
3413 c->rex_prefix = c->b;
3415 case 0xf0: /* LOCK */
3418 case 0xf2: /* REPNE/REPNZ */
3419 case 0xf3: /* REP/REPE/REPZ */
3420 c->rep_prefix = c->b;
3426 /* Any legacy prefix after a REX prefix nullifies its effect. */
3434 if (c->rex_prefix & 8)
3435 c->op_bytes = 8; /* REX.W */
3437 /* Opcode byte(s). */
3438 opcode = opcode_table[c->b];
3439 /* Two-byte opcode? */
3442 c->b = insn_fetch(u8, 1, c->eip);
3443 opcode = twobyte_table[c->b];
3445 c->d = opcode.flags;
3447 while (c->d & GroupMask) {
3448 switch (c->d & GroupMask) {
3450 c->modrm = insn_fetch(u8, 1, c->eip);
3452 goffset = (c->modrm >> 3) & 7;
3453 opcode = opcode.u.group[goffset];
3456 c->modrm = insn_fetch(u8, 1, c->eip);
3458 goffset = (c->modrm >> 3) & 7;
3459 if ((c->modrm >> 6) == 3)
3460 opcode = opcode.u.gdual->mod3[goffset];
3462 opcode = opcode.u.gdual->mod012[goffset];
3465 goffset = c->modrm & 7;
3466 opcode = opcode.u.group[goffset];
3469 if (c->rep_prefix && op_prefix)
3470 return X86EMUL_UNHANDLEABLE;
3471 simd_prefix = op_prefix ? 0x66 : c->rep_prefix;
3472 switch (simd_prefix) {
3473 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3474 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3475 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3476 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3480 return X86EMUL_UNHANDLEABLE;
3484 c->d |= opcode.flags;
3487 c->execute = opcode.u.execute;
3488 c->check_perm = opcode.check_perm;
3489 c->intercept = opcode.intercept;
3492 if (c->d == 0 || (c->d & Undefined))
3495 if (!(c->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3498 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
3501 if (c->d & Op3264) {
3502 if (mode == X86EMUL_MODE_PROT64)
3511 /* ModRM and SIB bytes. */
3513 rc = decode_modrm(ctxt, ops, &memop);
3514 if (!c->has_seg_override)
3515 set_seg_override(c, c->modrm_seg);
3516 } else if (c->d & MemAbs)
3517 rc = decode_abs(ctxt, ops, &memop);
3518 if (rc != X86EMUL_CONTINUE)
3521 if (!c->has_seg_override)
3522 set_seg_override(c, VCPU_SREG_DS);
3524 memop.addr.mem.seg = seg_override(ctxt, c);
3526 if (memop.type == OP_MEM && c->ad_bytes != 8)
3527 memop.addr.mem.ea = (u32)memop.addr.mem.ea;
3529 if (memop.type == OP_MEM && c->rip_relative)
3530 memop.addr.mem.ea += c->eip;
3533 * Decode and fetch the source operand: register, memory
3536 switch (c->d & SrcMask) {
3540 decode_register_operand(ctxt, &c->src, c, 0);
3549 memop.bytes = (c->d & ByteOp) ? 1 :
3555 rc = decode_imm(ctxt, &c->src, 2, false);
3558 rc = decode_imm(ctxt, &c->src, imm_size(c), true);
3561 rc = decode_imm(ctxt, &c->src, imm_size(c), false);
3564 rc = decode_imm(ctxt, &c->src, 1, true);
3567 rc = decode_imm(ctxt, &c->src, 1, false);
3570 c->src.type = OP_REG;
3571 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3572 c->src.addr.reg = &c->regs[VCPU_REGS_RAX];
3573 fetch_register_operand(&c->src);
3580 c->src.type = OP_MEM;
3581 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3582 c->src.addr.mem.ea =
3583 register_address(c, c->regs[VCPU_REGS_RSI]);
3584 c->src.addr.mem.seg = seg_override(ctxt, c);
3588 c->src.type = OP_IMM;
3589 c->src.addr.mem.ea = c->eip;
3590 c->src.bytes = c->op_bytes + 2;
3591 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
3594 memop.bytes = c->op_bytes + 2;
3599 if (rc != X86EMUL_CONTINUE)
3603 * Decode and fetch the second source operand: register, memory
3606 switch (c->d & Src2Mask) {
3611 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
3614 rc = decode_imm(ctxt, &c->src2, 1, true);
3621 rc = decode_imm(ctxt, &c->src2, imm_size(c), true);
3625 if (rc != X86EMUL_CONTINUE)
3628 /* Decode and fetch the destination operand: register or memory. */
3629 switch (c->d & DstMask) {
3631 decode_register_operand(ctxt, &c->dst, c,
3632 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
3635 c->dst.type = OP_IMM;
3636 c->dst.addr.mem.ea = c->eip;
3638 c->dst.val = insn_fetch(u8, 1, c->eip);
3643 if ((c->d & DstMask) == DstMem64)
3646 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3648 fetch_bit_operand(c);
3649 c->dst.orig_val = c->dst.val;
3652 c->dst.type = OP_REG;
3653 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3654 c->dst.addr.reg = &c->regs[VCPU_REGS_RAX];
3655 fetch_register_operand(&c->dst);
3656 c->dst.orig_val = c->dst.val;
3659 c->dst.type = OP_MEM;
3660 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3661 c->dst.addr.mem.ea =
3662 register_address(c, c->regs[VCPU_REGS_RDI]);
3663 c->dst.addr.mem.seg = VCPU_SREG_ES;
3667 /* Special instructions do their own operand decoding. */
3669 c->dst.type = OP_NONE; /* Disable writeback. */
3674 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3677 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3679 struct decode_cache *c = &ctxt->decode;
3681 /* The second termination condition only applies for REPE
3682 * and REPNE. Test if the repeat string operation prefix is
3683 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3684 * corresponding termination condition according to:
3685 * - if REPE/REPZ and ZF = 0 then done
3686 * - if REPNE/REPNZ and ZF = 1 then done
3688 if (((c->b == 0xa6) || (c->b == 0xa7) ||
3689 (c->b == 0xae) || (c->b == 0xaf))
3690 && (((c->rep_prefix == REPE_PREFIX) &&
3691 ((ctxt->eflags & EFLG_ZF) == 0))
3692 || ((c->rep_prefix == REPNE_PREFIX) &&
3693 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3700 x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3702 struct x86_emulate_ops *ops = ctxt->ops;
3704 struct decode_cache *c = &ctxt->decode;
3705 int rc = X86EMUL_CONTINUE;
3706 int saved_dst_type = c->dst.type;
3707 int irq; /* Used for int 3, int, and into */
3709 ctxt->decode.mem_read.pos = 0;
3711 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
3712 rc = emulate_ud(ctxt);
3716 /* LOCK prefix is allowed only with some instructions */
3717 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
3718 rc = emulate_ud(ctxt);
3722 if ((c->d & SrcMask) == SrcMemFAddr && c->src.type != OP_MEM) {
3723 rc = emulate_ud(ctxt);
3728 && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
3729 || !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
3730 rc = emulate_ud(ctxt);
3734 if ((c->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
3735 rc = emulate_nm(ctxt);
3739 if (unlikely(ctxt->guest_mode) && c->intercept) {
3740 rc = emulator_check_intercept(ctxt, c->intercept,
3741 X86_ICPT_PRE_EXCEPT);
3742 if (rc != X86EMUL_CONTINUE)
3746 /* Privileged instruction can be executed only in CPL=0 */
3747 if ((c->d & Priv) && ops->cpl(ctxt)) {
3748 rc = emulate_gp(ctxt, 0);
3752 /* Instruction can only be executed in protected mode */
3753 if ((c->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3754 rc = emulate_ud(ctxt);
3758 /* Do instruction specific permission checks */
3759 if (c->check_perm) {
3760 rc = c->check_perm(ctxt);
3761 if (rc != X86EMUL_CONTINUE)
3765 if (unlikely(ctxt->guest_mode) && c->intercept) {
3766 rc = emulator_check_intercept(ctxt, c->intercept,
3767 X86_ICPT_POST_EXCEPT);
3768 if (rc != X86EMUL_CONTINUE)
3772 if (c->rep_prefix && (c->d & String)) {
3773 /* All REP prefixes have the same first termination condition */
3774 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
3780 if ((c->src.type == OP_MEM) && !(c->d & NoAccess)) {
3781 rc = segmented_read(ctxt, c->src.addr.mem,
3782 c->src.valptr, c->src.bytes);
3783 if (rc != X86EMUL_CONTINUE)
3785 c->src.orig_val64 = c->src.val64;
3788 if (c->src2.type == OP_MEM) {
3789 rc = segmented_read(ctxt, c->src2.addr.mem,
3790 &c->src2.val, c->src2.bytes);
3791 if (rc != X86EMUL_CONTINUE)
3795 if ((c->d & DstMask) == ImplicitOps)
3799 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
3800 /* optimisation - avoid slow emulated read if Mov */
3801 rc = segmented_read(ctxt, c->dst.addr.mem,
3802 &c->dst.val, c->dst.bytes);
3803 if (rc != X86EMUL_CONTINUE)
3806 c->dst.orig_val = c->dst.val;
3810 if (unlikely(ctxt->guest_mode) && c->intercept) {
3811 rc = emulator_check_intercept(ctxt, c->intercept,
3812 X86_ICPT_POST_MEMACCESS);
3813 if (rc != X86EMUL_CONTINUE)
3818 rc = c->execute(ctxt);
3819 if (rc != X86EMUL_CONTINUE)
3828 case 0x06: /* push es */
3829 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
3831 case 0x07: /* pop es */
3832 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
3834 case 0x0e: /* push cs */
3835 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
3837 case 0x16: /* push ss */
3838 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
3840 case 0x17: /* pop ss */
3841 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
3843 case 0x1e: /* push ds */
3844 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
3846 case 0x1f: /* pop ds */
3847 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
3849 case 0x40 ... 0x47: /* inc r16/r32 */
3850 emulate_1op("inc", c->dst, ctxt->eflags);
3852 case 0x48 ... 0x4f: /* dec r16/r32 */
3853 emulate_1op("dec", c->dst, ctxt->eflags);
3855 case 0x63: /* movsxd */
3856 if (ctxt->mode != X86EMUL_MODE_PROT64)
3857 goto cannot_emulate;
3858 c->dst.val = (s32) c->src.val;
3860 case 0x6c: /* insb */
3861 case 0x6d: /* insw/insd */
3862 c->src.val = c->regs[VCPU_REGS_RDX];
3864 case 0x6e: /* outsb */
3865 case 0x6f: /* outsw/outsd */
3866 c->dst.val = c->regs[VCPU_REGS_RDX];
3869 case 0x70 ... 0x7f: /* jcc (short) */
3870 if (test_cc(c->b, ctxt->eflags))
3871 jmp_rel(c, c->src.val);
3875 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
3877 case 0x86 ... 0x87: /* xchg */
3879 /* Write back the register source. */
3880 c->src.val = c->dst.val;
3881 write_register_operand(&c->src);
3883 * Write back the memory destination with implicit LOCK
3886 c->dst.val = c->src.orig_val;
3889 case 0x8c: /* mov r/m, sreg */
3890 if (c->modrm_reg > VCPU_SREG_GS) {
3891 rc = emulate_ud(ctxt);
3894 c->dst.val = get_segment_selector(ctxt, c->modrm_reg);
3896 case 0x8d: /* lea r16/r32, m */
3897 c->dst.val = c->src.addr.mem.ea;
3899 case 0x8e: { /* mov seg, r/m16 */
3904 if (c->modrm_reg == VCPU_SREG_CS ||
3905 c->modrm_reg > VCPU_SREG_GS) {
3906 rc = emulate_ud(ctxt);
3910 if (c->modrm_reg == VCPU_SREG_SS)
3911 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3913 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
3915 c->dst.type = OP_NONE; /* Disable writeback. */
3918 case 0x8f: /* pop (sole member of Grp1a) */
3919 rc = emulate_grp1a(ctxt);
3921 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3922 if (c->dst.addr.reg == &c->regs[VCPU_REGS_RAX])
3925 case 0x98: /* cbw/cwde/cdqe */
3926 switch (c->op_bytes) {
3927 case 2: c->dst.val = (s8)c->dst.val; break;
3928 case 4: c->dst.val = (s16)c->dst.val; break;
3929 case 8: c->dst.val = (s32)c->dst.val; break;
3932 case 0xa8 ... 0xa9: /* test ax, imm */
3937 case 0xc3: /* ret */
3938 c->dst.type = OP_REG;
3939 c->dst.addr.reg = &c->eip;
3940 c->dst.bytes = c->op_bytes;
3943 case 0xc4: /* les */
3944 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_ES);
3946 case 0xc5: /* lds */
3947 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_DS);
3949 case 0xcb: /* ret far */
3950 rc = emulate_ret_far(ctxt, ops);
3952 case 0xcc: /* int3 */
3955 case 0xcd: /* int n */
3958 rc = emulate_int(ctxt, ops, irq);
3960 case 0xce: /* into */
3961 if (ctxt->eflags & EFLG_OF) {
3966 case 0xcf: /* iret */
3967 rc = emulate_iret(ctxt, ops);
3969 case 0xd0 ... 0xd1: /* Grp2 */
3972 case 0xd2 ... 0xd3: /* Grp2 */
3973 c->src.val = c->regs[VCPU_REGS_RCX];
3976 case 0xe0 ... 0xe2: /* loop/loopz/loopnz */
3977 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3978 if (address_mask(c, c->regs[VCPU_REGS_RCX]) != 0 &&
3979 (c->b == 0xe2 || test_cc(c->b ^ 0x5, ctxt->eflags)))
3980 jmp_rel(c, c->src.val);
3982 case 0xe3: /* jcxz/jecxz/jrcxz */
3983 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0)
3984 jmp_rel(c, c->src.val);
3986 case 0xe4: /* inb */
3989 case 0xe6: /* outb */
3990 case 0xe7: /* out */
3992 case 0xe8: /* call (near) */ {
3993 long int rel = c->src.val;
3994 c->src.val = (unsigned long) c->eip;
3999 case 0xe9: /* jmp rel */
4001 case 0xea: { /* jmp far */
4004 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
4006 rc = load_segment_descriptor(ctxt, ops, sel, VCPU_SREG_CS);
4007 if (rc != X86EMUL_CONTINUE)
4011 memcpy(&c->eip, c->src.valptr, c->op_bytes);
4015 jmp: /* jmp rel short */
4016 jmp_rel(c, c->src.val);
4017 c->dst.type = OP_NONE; /* Disable writeback. */
4019 case 0xec: /* in al,dx */
4020 case 0xed: /* in (e/r)ax,dx */
4021 c->src.val = c->regs[VCPU_REGS_RDX];
4023 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
4025 goto done; /* IO is needed */
4027 case 0xee: /* out dx,al */
4028 case 0xef: /* out dx,(e/r)ax */
4029 c->dst.val = c->regs[VCPU_REGS_RDX];
4031 ops->pio_out_emulated(ctxt, c->src.bytes, c->dst.val,
4033 c->dst.type = OP_NONE; /* Disable writeback. */
4035 case 0xf4: /* hlt */
4036 ctxt->ops->halt(ctxt);
4038 case 0xf5: /* cmc */
4039 /* complement carry flag from eflags reg */
4040 ctxt->eflags ^= EFLG_CF;
4042 case 0xf6 ... 0xf7: /* Grp3 */
4043 rc = emulate_grp3(ctxt, ops);
4045 case 0xf8: /* clc */
4046 ctxt->eflags &= ~EFLG_CF;
4048 case 0xf9: /* stc */
4049 ctxt->eflags |= EFLG_CF;
4051 case 0xfa: /* cli */
4052 if (emulator_bad_iopl(ctxt, ops)) {
4053 rc = emulate_gp(ctxt, 0);
4056 ctxt->eflags &= ~X86_EFLAGS_IF;
4058 case 0xfb: /* sti */
4059 if (emulator_bad_iopl(ctxt, ops)) {
4060 rc = emulate_gp(ctxt, 0);
4063 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
4064 ctxt->eflags |= X86_EFLAGS_IF;
4067 case 0xfc: /* cld */
4068 ctxt->eflags &= ~EFLG_DF;
4070 case 0xfd: /* std */
4071 ctxt->eflags |= EFLG_DF;
4073 case 0xfe: /* Grp4 */
4075 rc = emulate_grp45(ctxt);
4077 case 0xff: /* Grp5 */
4078 if (c->modrm_reg == 5)
4082 goto cannot_emulate;
4085 if (rc != X86EMUL_CONTINUE)
4089 rc = writeback(ctxt);
4090 if (rc != X86EMUL_CONTINUE)
4094 * restore dst type in case the decoding will be reused
4095 * (happens for string instruction )
4097 c->dst.type = saved_dst_type;
4099 if ((c->d & SrcMask) == SrcSI)
4100 string_addr_inc(ctxt, seg_override(ctxt, c),
4101 VCPU_REGS_RSI, &c->src);
4103 if ((c->d & DstMask) == DstDI)
4104 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4107 if (c->rep_prefix && (c->d & String)) {
4108 struct read_cache *r = &ctxt->decode.io_read;
4109 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
4111 if (!string_insn_completed(ctxt)) {
4113 * Re-enter guest when pio read ahead buffer is empty
4114 * or, if it is not used, after each 1024 iteration.
4116 if ((r->end != 0 || c->regs[VCPU_REGS_RCX] & 0x3ff) &&
4117 (r->end == 0 || r->end != r->pos)) {
4119 * Reset read cache. Usually happens before
4120 * decode, but since instruction is restarted
4121 * we have to do it here.
4123 ctxt->decode.mem_read.end = 0;
4124 return EMULATION_RESTART;
4126 goto done; /* skip rip writeback */
4133 if (rc == X86EMUL_PROPAGATE_FAULT)
4134 ctxt->have_exception = true;
4135 if (rc == X86EMUL_INTERCEPTED)
4136 return EMULATION_INTERCEPTED;
4138 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4142 case 0x05: /* syscall */
4143 rc = emulate_syscall(ctxt, ops);
4148 case 0x09: /* wbinvd */
4149 (ctxt->ops->wbinvd)(ctxt);
4151 case 0x08: /* invd */
4152 case 0x0d: /* GrpP (prefetch) */
4153 case 0x18: /* Grp16 (prefetch/nop) */
4155 case 0x20: /* mov cr, reg */
4156 c->dst.val = ops->get_cr(ctxt, c->modrm_reg);
4158 case 0x21: /* mov from dr to reg */
4159 ops->get_dr(ctxt, c->modrm_reg, &c->dst.val);
4161 case 0x22: /* mov reg, cr */
4162 if (ops->set_cr(ctxt, c->modrm_reg, c->src.val)) {
4163 emulate_gp(ctxt, 0);
4164 rc = X86EMUL_PROPAGATE_FAULT;
4167 c->dst.type = OP_NONE;
4169 case 0x23: /* mov from reg to dr */
4170 if (ops->set_dr(ctxt, c->modrm_reg, c->src.val &
4171 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4172 ~0ULL : ~0U)) < 0) {
4173 /* #UD condition is already handled by the code above */
4174 emulate_gp(ctxt, 0);
4175 rc = X86EMUL_PROPAGATE_FAULT;
4179 c->dst.type = OP_NONE; /* no writeback */
4183 msr_data = (u32)c->regs[VCPU_REGS_RAX]
4184 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
4185 if (ops->set_msr(ctxt, c->regs[VCPU_REGS_RCX], msr_data)) {
4186 emulate_gp(ctxt, 0);
4187 rc = X86EMUL_PROPAGATE_FAULT;
4190 rc = X86EMUL_CONTINUE;
4194 if (ops->get_msr(ctxt, c->regs[VCPU_REGS_RCX], &msr_data)) {
4195 emulate_gp(ctxt, 0);
4196 rc = X86EMUL_PROPAGATE_FAULT;
4199 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
4200 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
4202 rc = X86EMUL_CONTINUE;
4204 case 0x34: /* sysenter */
4205 rc = emulate_sysenter(ctxt, ops);
4207 case 0x35: /* sysexit */
4208 rc = emulate_sysexit(ctxt, ops);
4210 case 0x40 ... 0x4f: /* cmov */
4211 c->dst.val = c->dst.orig_val = c->src.val;
4212 if (!test_cc(c->b, ctxt->eflags))
4213 c->dst.type = OP_NONE; /* no writeback */
4215 case 0x80 ... 0x8f: /* jnz rel, etc*/
4216 if (test_cc(c->b, ctxt->eflags))
4217 jmp_rel(c, c->src.val);
4219 case 0x90 ... 0x9f: /* setcc r/m8 */
4220 c->dst.val = test_cc(c->b, ctxt->eflags);
4222 case 0xa0: /* push fs */
4223 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
4225 case 0xa1: /* pop fs */
4226 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
4230 c->dst.type = OP_NONE;
4231 /* only subword offset */
4232 c->src.val &= (c->dst.bytes << 3) - 1;
4233 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
4235 case 0xa4: /* shld imm8, r, r/m */
4236 case 0xa5: /* shld cl, r, r/m */
4237 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
4239 case 0xa8: /* push gs */
4240 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
4242 case 0xa9: /* pop gs */
4243 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
4247 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
4249 case 0xac: /* shrd imm8, r, r/m */
4250 case 0xad: /* shrd cl, r, r/m */
4251 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
4253 case 0xae: /* clflush */
4255 case 0xb0 ... 0xb1: /* cmpxchg */
4257 * Save real source value, then compare EAX against
4260 c->src.orig_val = c->src.val;
4261 c->src.val = c->regs[VCPU_REGS_RAX];
4262 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
4263 if (ctxt->eflags & EFLG_ZF) {
4264 /* Success: write back to memory. */
4265 c->dst.val = c->src.orig_val;
4267 /* Failure: write the value we saw to EAX. */
4268 c->dst.type = OP_REG;
4269 c->dst.addr.reg = (unsigned long *)&c->regs[VCPU_REGS_RAX];
4272 case 0xb2: /* lss */
4273 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_SS);
4277 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
4279 case 0xb4: /* lfs */
4280 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_FS);
4282 case 0xb5: /* lgs */
4283 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_GS);
4285 case 0xb6 ... 0xb7: /* movzx */
4286 c->dst.bytes = c->op_bytes;
4287 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
4290 case 0xba: /* Grp8 */
4291 switch (c->modrm_reg & 3) {
4304 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
4306 case 0xbc: { /* bsf */
4308 __asm__ ("bsf %2, %0; setz %1"
4309 : "=r"(c->dst.val), "=q"(zf)
4311 ctxt->eflags &= ~X86_EFLAGS_ZF;
4313 ctxt->eflags |= X86_EFLAGS_ZF;
4314 c->dst.type = OP_NONE; /* Disable writeback. */
4318 case 0xbd: { /* bsr */
4320 __asm__ ("bsr %2, %0; setz %1"
4321 : "=r"(c->dst.val), "=q"(zf)
4323 ctxt->eflags &= ~X86_EFLAGS_ZF;
4325 ctxt->eflags |= X86_EFLAGS_ZF;
4326 c->dst.type = OP_NONE; /* Disable writeback. */
4330 case 0xbe ... 0xbf: /* movsx */
4331 c->dst.bytes = c->op_bytes;
4332 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
4335 case 0xc0 ... 0xc1: /* xadd */
4336 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
4337 /* Write back the register source. */
4338 c->src.val = c->dst.orig_val;
4339 write_register_operand(&c->src);
4341 case 0xc3: /* movnti */
4342 c->dst.bytes = c->op_bytes;
4343 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
4346 case 0xc7: /* Grp9 (cmpxchg8b) */
4347 rc = emulate_grp9(ctxt, ops);
4350 goto cannot_emulate;
4353 if (rc != X86EMUL_CONTINUE)
4359 return EMULATION_FAILED;