1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
32 * Opcode effective-address decode tables.
33 * Note that we only emulate instructions that have at least one memory
34 * operand (excluding implicit stack references). We assume that stack
35 * references and instruction fetches will never occur in special memory
36 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
40 /* Operand sizes: 8-bit operands or specified/overridden size. */
41 #define ByteOp (1<<0) /* 8-bit operands. */
42 /* Destination operand type. */
43 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
44 #define DstReg (2<<1) /* Register operand. */
45 #define DstMem (3<<1) /* Memory operand. */
46 #define DstAcc (4<<1) /* Destination Accumulator */
47 #define DstDI (5<<1) /* Destination is in ES:(E)DI */
48 #define DstMem64 (6<<1) /* 64bit memory operand */
49 #define DstImmUByte (7<<1) /* 8-bit unsigned immediate operand */
50 #define DstMask (7<<1)
51 /* Source operand type. */
52 #define SrcNone (0<<4) /* No source operand. */
53 #define SrcReg (1<<4) /* Register operand. */
54 #define SrcMem (2<<4) /* Memory operand. */
55 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
56 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
57 #define SrcImm (5<<4) /* Immediate operand. */
58 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
59 #define SrcOne (7<<4) /* Implied '1' */
60 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
61 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
62 #define SrcSI (0xa<<4) /* Source is in the DS:RSI */
63 #define SrcImmFAddr (0xb<<4) /* Source is immediate far address */
64 #define SrcMemFAddr (0xc<<4) /* Source is far address in memory */
65 #define SrcAcc (0xd<<4) /* Source Accumulator */
66 #define SrcImmU16 (0xe<<4) /* Immediate operand, unsigned, 16 bits */
67 #define SrcMask (0xf<<4)
68 /* Generic ModRM decode. */
70 /* Destination is only written; never read. */
73 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
74 #define String (1<<12) /* String instruction (rep capable) */
75 #define Stack (1<<13) /* Stack instruction (push/pop) */
76 #define GroupMask (7<<14) /* Opcode uses one of the group mechanisms */
77 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
78 #define GroupDual (2<<14) /* Alternate decoding of mod == 3 */
79 #define Prefix (3<<14) /* Instruction varies with 66/f2/f3 prefix */
80 #define RMExt (4<<14) /* Opcode extension in ModRM r/m if mod == 3 */
81 #define Sse (1<<17) /* SSE Vector instruction */
83 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
84 #define VendorSpecific (1<<22) /* Vendor specific instruction */
85 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
86 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
87 #define Undefined (1<<25) /* No Such Instruction */
88 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
89 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
91 /* Source 2 operand type */
92 #define Src2None (0<<29)
93 #define Src2CL (1<<29)
94 #define Src2ImmByte (2<<29)
95 #define Src2One (3<<29)
96 #define Src2Imm (4<<29)
97 #define Src2Mask (7<<29)
100 #define X3(x...) X2(x), x
101 #define X4(x...) X2(x), X2(x)
102 #define X5(x...) X4(x), x
103 #define X6(x...) X4(x), X2(x)
104 #define X7(x...) X4(x), X3(x)
105 #define X8(x...) X4(x), X4(x)
106 #define X16(x...) X8(x), X8(x)
112 int (*execute)(struct x86_emulate_ctxt *ctxt);
113 struct opcode *group;
114 struct group_dual *gdual;
115 struct gprefix *gprefix;
117 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
121 struct opcode mod012[8];
122 struct opcode mod3[8];
126 struct opcode pfx_no;
127 struct opcode pfx_66;
128 struct opcode pfx_f2;
129 struct opcode pfx_f3;
132 /* EFLAGS bit definitions. */
133 #define EFLG_ID (1<<21)
134 #define EFLG_VIP (1<<20)
135 #define EFLG_VIF (1<<19)
136 #define EFLG_AC (1<<18)
137 #define EFLG_VM (1<<17)
138 #define EFLG_RF (1<<16)
139 #define EFLG_IOPL (3<<12)
140 #define EFLG_NT (1<<14)
141 #define EFLG_OF (1<<11)
142 #define EFLG_DF (1<<10)
143 #define EFLG_IF (1<<9)
144 #define EFLG_TF (1<<8)
145 #define EFLG_SF (1<<7)
146 #define EFLG_ZF (1<<6)
147 #define EFLG_AF (1<<4)
148 #define EFLG_PF (1<<2)
149 #define EFLG_CF (1<<0)
151 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
152 #define EFLG_RESERVED_ONE_MASK 2
155 * Instruction emulation:
156 * Most instructions are emulated directly via a fragment of inline assembly
157 * code. This allows us to save/restore EFLAGS and thus very easily pick up
158 * any modified flags.
161 #if defined(CONFIG_X86_64)
162 #define _LO32 "k" /* force 32-bit operand */
163 #define _STK "%%rsp" /* stack pointer */
164 #elif defined(__i386__)
165 #define _LO32 "" /* force 32-bit operand */
166 #define _STK "%%esp" /* stack pointer */
170 * These EFLAGS bits are restored from saved value during emulation, and
171 * any changes are written back to the saved value after emulation.
173 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
175 /* Before executing instruction: restore necessary bits in EFLAGS. */
176 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
177 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
178 "movl %"_sav",%"_LO32 _tmp"; " \
181 "movl %"_msk",%"_LO32 _tmp"; " \
182 "andl %"_LO32 _tmp",("_STK"); " \
184 "notl %"_LO32 _tmp"; " \
185 "andl %"_LO32 _tmp",("_STK"); " \
186 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
188 "orl %"_LO32 _tmp",("_STK"); " \
192 /* After executing instruction: write-back necessary bits in EFLAGS. */
193 #define _POST_EFLAGS(_sav, _msk, _tmp) \
194 /* _sav |= EFLAGS & _msk; */ \
197 "andl %"_msk",%"_LO32 _tmp"; " \
198 "orl %"_LO32 _tmp",%"_sav"; "
206 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix, _dsttype) \
208 __asm__ __volatile__ ( \
209 _PRE_EFLAGS("0", "4", "2") \
210 _op _suffix " %"_x"3,%1; " \
211 _POST_EFLAGS("0", "4", "2") \
212 : "=m" (_eflags), "+q" (*(_dsttype*)&(_dst).val),\
214 : _y ((_src).val), "i" (EFLAGS_MASK)); \
218 /* Raw emulation: instruction has two explicit operands. */
219 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
221 unsigned long _tmp; \
223 switch ((_dst).bytes) { \
225 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w",u16);\
228 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l",u32);\
231 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q",u64)); \
236 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
238 unsigned long _tmp; \
239 switch ((_dst).bytes) { \
241 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b",u8); \
244 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
245 _wx, _wy, _lx, _ly, _qx, _qy); \
250 /* Source operand is byte-sized and may be restricted to just %cl. */
251 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
252 __emulate_2op(_op, _src, _dst, _eflags, \
253 "b", "c", "b", "c", "b", "c", "b", "c")
255 /* Source operand is byte, word, long or quad sized. */
256 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
257 __emulate_2op(_op, _src, _dst, _eflags, \
258 "b", "q", "w", "r", _LO32, "r", "", "r")
260 /* Source operand is word, long or quad sized. */
261 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
262 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
263 "w", "r", _LO32, "r", "", "r")
265 /* Instruction has three operands and one operand is stored in ECX register */
266 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
268 unsigned long _tmp; \
269 _type _clv = (_cl).val; \
270 _type _srcv = (_src).val; \
271 _type _dstv = (_dst).val; \
273 __asm__ __volatile__ ( \
274 _PRE_EFLAGS("0", "5", "2") \
275 _op _suffix " %4,%1 \n" \
276 _POST_EFLAGS("0", "5", "2") \
277 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
278 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
281 (_cl).val = (unsigned long) _clv; \
282 (_src).val = (unsigned long) _srcv; \
283 (_dst).val = (unsigned long) _dstv; \
286 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
288 switch ((_dst).bytes) { \
290 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
291 "w", unsigned short); \
294 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
295 "l", unsigned int); \
298 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
299 "q", unsigned long)); \
304 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
306 unsigned long _tmp; \
308 __asm__ __volatile__ ( \
309 _PRE_EFLAGS("0", "3", "2") \
310 _op _suffix " %1; " \
311 _POST_EFLAGS("0", "3", "2") \
312 : "=m" (_eflags), "+m" ((_dst).val), \
314 : "i" (EFLAGS_MASK)); \
317 /* Instruction has only one explicit operand (no source operand). */
318 #define emulate_1op(_op, _dst, _eflags) \
320 switch ((_dst).bytes) { \
321 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
322 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
323 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
324 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
328 #define __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, _suffix) \
330 unsigned long _tmp; \
332 __asm__ __volatile__ ( \
333 _PRE_EFLAGS("0", "4", "1") \
334 _op _suffix " %5; " \
335 _POST_EFLAGS("0", "4", "1") \
336 : "=m" (_eflags), "=&r" (_tmp), \
337 "+a" (_rax), "+d" (_rdx) \
338 : "i" (EFLAGS_MASK), "m" ((_src).val), \
339 "a" (_rax), "d" (_rdx)); \
342 #define __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _suffix, _ex) \
344 unsigned long _tmp; \
346 __asm__ __volatile__ ( \
347 _PRE_EFLAGS("0", "5", "1") \
349 _op _suffix " %6; " \
351 _POST_EFLAGS("0", "5", "1") \
352 ".pushsection .fixup,\"ax\" \n\t" \
353 "3: movb $1, %4 \n\t" \
356 _ASM_EXTABLE(1b, 3b) \
357 : "=m" (_eflags), "=&r" (_tmp), \
358 "+a" (_rax), "+d" (_rdx), "+qm"(_ex) \
359 : "i" (EFLAGS_MASK), "m" ((_src).val), \
360 "a" (_rax), "d" (_rdx)); \
363 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
364 #define emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags) \
366 switch((_src).bytes) { \
368 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
372 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
376 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
380 ON64(__emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
386 #define emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _ex) \
388 switch((_src).bytes) { \
390 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
391 _eflags, "b", _ex); \
394 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
395 _eflags, "w", _ex); \
398 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
399 _eflags, "l", _ex); \
402 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
403 _eflags, "q", _ex)); \
408 /* Fetch next part of the instruction being emulated. */
409 #define insn_fetch(_type, _size, _eip) \
410 ({ unsigned long _x; \
411 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
412 if (rc != X86EMUL_CONTINUE) \
418 #define insn_fetch_arr(_arr, _size, _eip) \
419 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
420 if (rc != X86EMUL_CONTINUE) \
425 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
426 enum x86_intercept intercept,
427 enum x86_intercept_stage stage)
429 struct x86_instruction_info info = {
430 .intercept = intercept,
431 .rep_prefix = ctxt->decode.rep_prefix,
432 .modrm_mod = ctxt->decode.modrm_mod,
433 .modrm_reg = ctxt->decode.modrm_reg,
434 .modrm_rm = ctxt->decode.modrm_rm,
435 .src_val = ctxt->decode.src.val64,
436 .src_bytes = ctxt->decode.src.bytes,
437 .dst_bytes = ctxt->decode.dst.bytes,
438 .ad_bytes = ctxt->decode.ad_bytes,
439 .next_rip = ctxt->eip,
442 return ctxt->ops->intercept(ctxt, &info, stage);
445 static inline unsigned long ad_mask(struct decode_cache *c)
447 return (1UL << (c->ad_bytes << 3)) - 1;
450 /* Access/update address held in a register, based on addressing mode. */
451 static inline unsigned long
452 address_mask(struct decode_cache *c, unsigned long reg)
454 if (c->ad_bytes == sizeof(unsigned long))
457 return reg & ad_mask(c);
460 static inline unsigned long
461 register_address(struct decode_cache *c, unsigned long reg)
463 return address_mask(c, reg);
467 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
469 if (c->ad_bytes == sizeof(unsigned long))
472 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
475 static inline void jmp_rel(struct decode_cache *c, int rel)
477 register_address_increment(c, &c->eip, rel);
480 static u32 desc_limit_scaled(struct desc_struct *desc)
482 u32 limit = get_desc_limit(desc);
484 return desc->g ? (limit << 12) | 0xfff : limit;
487 static void set_seg_override(struct decode_cache *c, int seg)
489 c->has_seg_override = true;
490 c->seg_override = seg;
493 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
494 struct x86_emulate_ops *ops, int seg)
496 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
499 return ops->get_cached_segment_base(ctxt, seg);
502 static unsigned seg_override(struct x86_emulate_ctxt *ctxt,
503 struct x86_emulate_ops *ops,
504 struct decode_cache *c)
506 if (!c->has_seg_override)
509 return c->seg_override;
512 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
513 u32 error, bool valid)
515 ctxt->exception.vector = vec;
516 ctxt->exception.error_code = error;
517 ctxt->exception.error_code_valid = valid;
518 return X86EMUL_PROPAGATE_FAULT;
521 static int emulate_db(struct x86_emulate_ctxt *ctxt)
523 return emulate_exception(ctxt, DB_VECTOR, 0, false);
526 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
528 return emulate_exception(ctxt, GP_VECTOR, err, true);
531 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
533 return emulate_exception(ctxt, SS_VECTOR, err, true);
536 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
538 return emulate_exception(ctxt, UD_VECTOR, 0, false);
541 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
543 return emulate_exception(ctxt, TS_VECTOR, err, true);
546 static int emulate_de(struct x86_emulate_ctxt *ctxt)
548 return emulate_exception(ctxt, DE_VECTOR, 0, false);
551 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
553 return emulate_exception(ctxt, NM_VECTOR, 0, false);
556 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
559 struct desc_struct desc;
561 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
565 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
570 struct desc_struct desc;
572 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
573 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
576 static int __linearize(struct x86_emulate_ctxt *ctxt,
577 struct segmented_address addr,
578 unsigned size, bool write, bool fetch,
581 struct decode_cache *c = &ctxt->decode;
582 struct desc_struct desc;
589 la = seg_base(ctxt, ctxt->ops, addr.seg) + addr.ea;
590 switch (ctxt->mode) {
591 case X86EMUL_MODE_REAL:
593 case X86EMUL_MODE_PROT64:
594 if (((signed long)la << 16) >> 16 != la)
595 return emulate_gp(ctxt, 0);
598 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
602 /* code segment or read-only data segment */
603 if (((desc.type & 8) || !(desc.type & 2)) && write)
605 /* unreadable code segment */
606 if (!fetch && (desc.type & 8) && !(desc.type & 2))
608 lim = desc_limit_scaled(&desc);
609 if ((desc.type & 8) || !(desc.type & 4)) {
610 /* expand-up segment */
611 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
614 /* exapand-down segment */
615 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
617 lim = desc.d ? 0xffffffff : 0xffff;
618 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
621 cpl = ctxt->ops->cpl(ctxt);
624 if (!(desc.type & 8)) {
628 } else if ((desc.type & 8) && !(desc.type & 4)) {
629 /* nonconforming code segment */
632 } else if ((desc.type & 8) && (desc.type & 4)) {
633 /* conforming code segment */
639 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : c->ad_bytes != 8)
642 return X86EMUL_CONTINUE;
644 if (addr.seg == VCPU_SREG_SS)
645 return emulate_ss(ctxt, addr.seg);
647 return emulate_gp(ctxt, addr.seg);
650 static int linearize(struct x86_emulate_ctxt *ctxt,
651 struct segmented_address addr,
652 unsigned size, bool write,
655 return __linearize(ctxt, addr, size, write, false, linear);
659 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
660 struct segmented_address addr,
667 rc = linearize(ctxt, addr, size, false, &linear);
668 if (rc != X86EMUL_CONTINUE)
670 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
673 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
674 struct x86_emulate_ops *ops,
675 unsigned long eip, u8 *dest)
677 struct fetch_cache *fc = &ctxt->decode.fetch;
681 if (eip == fc->end) {
682 unsigned long linear;
683 struct segmented_address addr = { .seg=VCPU_SREG_CS, .ea=eip};
684 cur_size = fc->end - fc->start;
685 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
686 rc = __linearize(ctxt, addr, size, false, true, &linear);
687 if (rc != X86EMUL_CONTINUE)
689 rc = ops->fetch(ctxt, linear, fc->data + cur_size,
690 size, &ctxt->exception);
691 if (rc != X86EMUL_CONTINUE)
695 *dest = fc->data[eip - fc->start];
696 return X86EMUL_CONTINUE;
699 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
700 struct x86_emulate_ops *ops,
701 unsigned long eip, void *dest, unsigned size)
705 /* x86 instructions are limited to 15 bytes. */
706 if (eip + size - ctxt->eip > 15)
707 return X86EMUL_UNHANDLEABLE;
709 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
710 if (rc != X86EMUL_CONTINUE)
713 return X86EMUL_CONTINUE;
717 * Given the 'reg' portion of a ModRM byte, and a register block, return a
718 * pointer into the block that addresses the relevant register.
719 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
721 static void *decode_register(u8 modrm_reg, unsigned long *regs,
726 p = ®s[modrm_reg];
727 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
728 p = (unsigned char *)®s[modrm_reg & 3] + 1;
732 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
733 struct x86_emulate_ops *ops,
734 struct segmented_address addr,
735 u16 *size, unsigned long *address, int op_bytes)
742 rc = segmented_read_std(ctxt, addr, size, 2);
743 if (rc != X86EMUL_CONTINUE)
746 rc = segmented_read_std(ctxt, addr, address, op_bytes);
750 static int test_cc(unsigned int condition, unsigned int flags)
754 switch ((condition & 15) >> 1) {
756 rc |= (flags & EFLG_OF);
758 case 1: /* b/c/nae */
759 rc |= (flags & EFLG_CF);
762 rc |= (flags & EFLG_ZF);
765 rc |= (flags & (EFLG_CF|EFLG_ZF));
768 rc |= (flags & EFLG_SF);
771 rc |= (flags & EFLG_PF);
774 rc |= (flags & EFLG_ZF);
777 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
781 /* Odd condition identifiers (lsb == 1) have inverted sense. */
782 return (!!rc ^ (condition & 1));
785 static void fetch_register_operand(struct operand *op)
789 op->val = *(u8 *)op->addr.reg;
792 op->val = *(u16 *)op->addr.reg;
795 op->val = *(u32 *)op->addr.reg;
798 op->val = *(u64 *)op->addr.reg;
803 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
805 ctxt->ops->get_fpu(ctxt);
807 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
808 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
809 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
810 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
811 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
812 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
813 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
814 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
816 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
817 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
818 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
819 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
820 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
821 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
822 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
823 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
827 ctxt->ops->put_fpu(ctxt);
830 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
833 ctxt->ops->get_fpu(ctxt);
835 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
836 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
837 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
838 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
839 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
840 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
841 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
842 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
844 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
845 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
846 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
847 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
848 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
849 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
850 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
851 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
855 ctxt->ops->put_fpu(ctxt);
858 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
860 struct decode_cache *c,
863 unsigned reg = c->modrm_reg;
864 int highbyte_regs = c->rex_prefix == 0;
867 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
873 read_sse_reg(ctxt, &op->vec_val, reg);
878 if ((c->d & ByteOp) && !inhibit_bytereg) {
879 op->addr.reg = decode_register(reg, c->regs, highbyte_regs);
882 op->addr.reg = decode_register(reg, c->regs, 0);
883 op->bytes = c->op_bytes;
885 fetch_register_operand(op);
886 op->orig_val = op->val;
889 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
890 struct x86_emulate_ops *ops,
893 struct decode_cache *c = &ctxt->decode;
895 int index_reg = 0, base_reg = 0, scale;
896 int rc = X86EMUL_CONTINUE;
900 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
901 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
902 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
905 c->modrm = insn_fetch(u8, 1, c->eip);
906 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
907 c->modrm_reg |= (c->modrm & 0x38) >> 3;
908 c->modrm_rm |= (c->modrm & 0x07);
909 c->modrm_seg = VCPU_SREG_DS;
911 if (c->modrm_mod == 3) {
913 op->bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
914 op->addr.reg = decode_register(c->modrm_rm,
915 c->regs, c->d & ByteOp);
919 op->addr.xmm = c->modrm_rm;
920 read_sse_reg(ctxt, &op->vec_val, c->modrm_rm);
923 fetch_register_operand(op);
929 if (c->ad_bytes == 2) {
930 unsigned bx = c->regs[VCPU_REGS_RBX];
931 unsigned bp = c->regs[VCPU_REGS_RBP];
932 unsigned si = c->regs[VCPU_REGS_RSI];
933 unsigned di = c->regs[VCPU_REGS_RDI];
935 /* 16-bit ModR/M decode. */
936 switch (c->modrm_mod) {
938 if (c->modrm_rm == 6)
939 modrm_ea += insn_fetch(u16, 2, c->eip);
942 modrm_ea += insn_fetch(s8, 1, c->eip);
945 modrm_ea += insn_fetch(u16, 2, c->eip);
948 switch (c->modrm_rm) {
968 if (c->modrm_mod != 0)
975 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
976 (c->modrm_rm == 6 && c->modrm_mod != 0))
977 c->modrm_seg = VCPU_SREG_SS;
978 modrm_ea = (u16)modrm_ea;
980 /* 32/64-bit ModR/M decode. */
981 if ((c->modrm_rm & 7) == 4) {
982 sib = insn_fetch(u8, 1, c->eip);
983 index_reg |= (sib >> 3) & 7;
987 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
988 modrm_ea += insn_fetch(s32, 4, c->eip);
990 modrm_ea += c->regs[base_reg];
992 modrm_ea += c->regs[index_reg] << scale;
993 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
994 if (ctxt->mode == X86EMUL_MODE_PROT64)
997 modrm_ea += c->regs[c->modrm_rm];
998 switch (c->modrm_mod) {
1000 if (c->modrm_rm == 5)
1001 modrm_ea += insn_fetch(s32, 4, c->eip);
1004 modrm_ea += insn_fetch(s8, 1, c->eip);
1007 modrm_ea += insn_fetch(s32, 4, c->eip);
1011 op->addr.mem.ea = modrm_ea;
1016 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1017 struct x86_emulate_ops *ops,
1020 struct decode_cache *c = &ctxt->decode;
1021 int rc = X86EMUL_CONTINUE;
1024 switch (c->ad_bytes) {
1026 op->addr.mem.ea = insn_fetch(u16, 2, c->eip);
1029 op->addr.mem.ea = insn_fetch(u32, 4, c->eip);
1032 op->addr.mem.ea = insn_fetch(u64, 8, c->eip);
1039 static void fetch_bit_operand(struct decode_cache *c)
1043 if (c->dst.type == OP_MEM && c->src.type == OP_REG) {
1044 mask = ~(c->dst.bytes * 8 - 1);
1046 if (c->src.bytes == 2)
1047 sv = (s16)c->src.val & (s16)mask;
1048 else if (c->src.bytes == 4)
1049 sv = (s32)c->src.val & (s32)mask;
1051 c->dst.addr.mem.ea += (sv >> 3);
1054 /* only subword offset */
1055 c->src.val &= (c->dst.bytes << 3) - 1;
1058 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1059 struct x86_emulate_ops *ops,
1060 unsigned long addr, void *dest, unsigned size)
1063 struct read_cache *mc = &ctxt->decode.mem_read;
1066 int n = min(size, 8u);
1068 if (mc->pos < mc->end)
1071 rc = ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1073 if (rc != X86EMUL_CONTINUE)
1078 memcpy(dest, mc->data + mc->pos, n);
1083 return X86EMUL_CONTINUE;
1086 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1087 struct segmented_address addr,
1094 rc = linearize(ctxt, addr, size, false, &linear);
1095 if (rc != X86EMUL_CONTINUE)
1097 return read_emulated(ctxt, ctxt->ops, linear, data, size);
1100 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1101 struct segmented_address addr,
1108 rc = linearize(ctxt, addr, size, true, &linear);
1109 if (rc != X86EMUL_CONTINUE)
1111 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1115 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1116 struct segmented_address addr,
1117 const void *orig_data, const void *data,
1123 rc = linearize(ctxt, addr, size, true, &linear);
1124 if (rc != X86EMUL_CONTINUE)
1126 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1127 size, &ctxt->exception);
1130 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1131 struct x86_emulate_ops *ops,
1132 unsigned int size, unsigned short port,
1135 struct read_cache *rc = &ctxt->decode.io_read;
1137 if (rc->pos == rc->end) { /* refill pio read ahead */
1138 struct decode_cache *c = &ctxt->decode;
1139 unsigned int in_page, n;
1140 unsigned int count = c->rep_prefix ?
1141 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
1142 in_page = (ctxt->eflags & EFLG_DF) ?
1143 offset_in_page(c->regs[VCPU_REGS_RDI]) :
1144 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
1145 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1149 rc->pos = rc->end = 0;
1150 if (!ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1155 memcpy(dest, rc->data + rc->pos, size);
1160 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1161 struct x86_emulate_ops *ops,
1162 u16 selector, struct desc_ptr *dt)
1164 if (selector & 1 << 2) {
1165 struct desc_struct desc;
1168 memset (dt, 0, sizeof *dt);
1169 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1172 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1173 dt->address = get_desc_base(&desc);
1175 ops->get_gdt(ctxt, dt);
1178 /* allowed just for 8 bytes segments */
1179 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1180 struct x86_emulate_ops *ops,
1181 u16 selector, struct desc_struct *desc)
1184 u16 index = selector >> 3;
1188 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1190 if (dt.size < index * 8 + 7)
1191 return emulate_gp(ctxt, selector & 0xfffc);
1192 addr = dt.address + index * 8;
1193 ret = ops->read_std(ctxt, addr, desc, sizeof *desc, &ctxt->exception);
1198 /* allowed just for 8 bytes segments */
1199 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1200 struct x86_emulate_ops *ops,
1201 u16 selector, struct desc_struct *desc)
1204 u16 index = selector >> 3;
1208 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1210 if (dt.size < index * 8 + 7)
1211 return emulate_gp(ctxt, selector & 0xfffc);
1213 addr = dt.address + index * 8;
1214 ret = ops->write_std(ctxt, addr, desc, sizeof *desc, &ctxt->exception);
1219 /* Does not support long mode */
1220 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1221 struct x86_emulate_ops *ops,
1222 u16 selector, int seg)
1224 struct desc_struct seg_desc;
1226 unsigned err_vec = GP_VECTOR;
1228 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1231 memset(&seg_desc, 0, sizeof seg_desc);
1233 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1234 || ctxt->mode == X86EMUL_MODE_REAL) {
1235 /* set real mode segment descriptor */
1236 set_desc_base(&seg_desc, selector << 4);
1237 set_desc_limit(&seg_desc, 0xffff);
1244 /* NULL selector is not valid for TR, CS and SS */
1245 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1249 /* TR should be in GDT only */
1250 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1253 if (null_selector) /* for NULL selector skip all following checks */
1256 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1257 if (ret != X86EMUL_CONTINUE)
1260 err_code = selector & 0xfffc;
1261 err_vec = GP_VECTOR;
1263 /* can't load system descriptor into segment selecor */
1264 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1268 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1274 cpl = ops->cpl(ctxt);
1279 * segment is not a writable data segment or segment
1280 * selector's RPL != CPL or segment selector's RPL != CPL
1282 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1286 if (!(seg_desc.type & 8))
1289 if (seg_desc.type & 4) {
1295 if (rpl > cpl || dpl != cpl)
1298 /* CS(RPL) <- CPL */
1299 selector = (selector & 0xfffc) | cpl;
1302 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1305 case VCPU_SREG_LDTR:
1306 if (seg_desc.s || seg_desc.type != 2)
1309 default: /* DS, ES, FS, or GS */
1311 * segment is not a data or readable code segment or
1312 * ((segment is a data or nonconforming code segment)
1313 * and (both RPL and CPL > DPL))
1315 if ((seg_desc.type & 0xa) == 0x8 ||
1316 (((seg_desc.type & 0xc) != 0xc) &&
1317 (rpl > dpl && cpl > dpl)))
1323 /* mark segment as accessed */
1325 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1326 if (ret != X86EMUL_CONTINUE)
1330 ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1331 return X86EMUL_CONTINUE;
1333 emulate_exception(ctxt, err_vec, err_code, true);
1334 return X86EMUL_PROPAGATE_FAULT;
1337 static void write_register_operand(struct operand *op)
1339 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1340 switch (op->bytes) {
1342 *(u8 *)op->addr.reg = (u8)op->val;
1345 *(u16 *)op->addr.reg = (u16)op->val;
1348 *op->addr.reg = (u32)op->val;
1349 break; /* 64b: zero-extend */
1351 *op->addr.reg = op->val;
1356 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1357 struct x86_emulate_ops *ops)
1360 struct decode_cache *c = &ctxt->decode;
1362 switch (c->dst.type) {
1364 write_register_operand(&c->dst);
1368 rc = segmented_cmpxchg(ctxt,
1374 rc = segmented_write(ctxt,
1378 if (rc != X86EMUL_CONTINUE)
1382 write_sse_reg(ctxt, &c->dst.vec_val, c->dst.addr.xmm);
1390 return X86EMUL_CONTINUE;
1393 static int em_push(struct x86_emulate_ctxt *ctxt)
1395 struct decode_cache *c = &ctxt->decode;
1396 struct segmented_address addr;
1398 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1399 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1400 addr.seg = VCPU_SREG_SS;
1402 /* Disable writeback. */
1403 c->dst.type = OP_NONE;
1404 return segmented_write(ctxt, addr, &c->src.val, c->op_bytes);
1407 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1408 struct x86_emulate_ops *ops,
1409 void *dest, int len)
1411 struct decode_cache *c = &ctxt->decode;
1413 struct segmented_address addr;
1415 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1416 addr.seg = VCPU_SREG_SS;
1417 rc = segmented_read(ctxt, addr, dest, len);
1418 if (rc != X86EMUL_CONTINUE)
1421 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1425 static int em_pop(struct x86_emulate_ctxt *ctxt)
1427 struct decode_cache *c = &ctxt->decode;
1429 return emulate_pop(ctxt, ctxt->ops, &c->dst.val, c->op_bytes);
1432 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1433 struct x86_emulate_ops *ops,
1434 void *dest, int len)
1437 unsigned long val, change_mask;
1438 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1439 int cpl = ops->cpl(ctxt);
1441 rc = emulate_pop(ctxt, ops, &val, len);
1442 if (rc != X86EMUL_CONTINUE)
1445 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1446 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1448 switch(ctxt->mode) {
1449 case X86EMUL_MODE_PROT64:
1450 case X86EMUL_MODE_PROT32:
1451 case X86EMUL_MODE_PROT16:
1453 change_mask |= EFLG_IOPL;
1455 change_mask |= EFLG_IF;
1457 case X86EMUL_MODE_VM86:
1459 return emulate_gp(ctxt, 0);
1460 change_mask |= EFLG_IF;
1462 default: /* real mode */
1463 change_mask |= (EFLG_IOPL | EFLG_IF);
1467 *(unsigned long *)dest =
1468 (ctxt->eflags & ~change_mask) | (val & change_mask);
1473 static int em_popf(struct x86_emulate_ctxt *ctxt)
1475 struct decode_cache *c = &ctxt->decode;
1477 c->dst.type = OP_REG;
1478 c->dst.addr.reg = &ctxt->eflags;
1479 c->dst.bytes = c->op_bytes;
1480 return emulate_popf(ctxt, ctxt->ops, &c->dst.val, c->op_bytes);
1483 static int emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1484 struct x86_emulate_ops *ops, int seg)
1486 struct decode_cache *c = &ctxt->decode;
1488 c->src.val = get_segment_selector(ctxt, seg);
1490 return em_push(ctxt);
1493 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1494 struct x86_emulate_ops *ops, int seg)
1496 struct decode_cache *c = &ctxt->decode;
1497 unsigned long selector;
1500 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1501 if (rc != X86EMUL_CONTINUE)
1504 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1508 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1510 struct decode_cache *c = &ctxt->decode;
1511 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1512 int rc = X86EMUL_CONTINUE;
1513 int reg = VCPU_REGS_RAX;
1515 while (reg <= VCPU_REGS_RDI) {
1516 (reg == VCPU_REGS_RSP) ?
1517 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1520 if (rc != X86EMUL_CONTINUE)
1529 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1531 struct decode_cache *c = &ctxt->decode;
1533 c->src.val = (unsigned long)ctxt->eflags;
1534 return em_push(ctxt);
1537 static int em_popa(struct x86_emulate_ctxt *ctxt)
1539 struct decode_cache *c = &ctxt->decode;
1540 int rc = X86EMUL_CONTINUE;
1541 int reg = VCPU_REGS_RDI;
1543 while (reg >= VCPU_REGS_RAX) {
1544 if (reg == VCPU_REGS_RSP) {
1545 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1550 rc = emulate_pop(ctxt, ctxt->ops, &c->regs[reg], c->op_bytes);
1551 if (rc != X86EMUL_CONTINUE)
1558 int emulate_int_real(struct x86_emulate_ctxt *ctxt,
1559 struct x86_emulate_ops *ops, int irq)
1561 struct decode_cache *c = &ctxt->decode;
1568 /* TODO: Add limit checks */
1569 c->src.val = ctxt->eflags;
1571 if (rc != X86EMUL_CONTINUE)
1574 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1576 c->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1578 if (rc != X86EMUL_CONTINUE)
1581 c->src.val = c->eip;
1583 if (rc != X86EMUL_CONTINUE)
1586 ops->get_idt(ctxt, &dt);
1588 eip_addr = dt.address + (irq << 2);
1589 cs_addr = dt.address + (irq << 2) + 2;
1591 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1592 if (rc != X86EMUL_CONTINUE)
1595 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1596 if (rc != X86EMUL_CONTINUE)
1599 rc = load_segment_descriptor(ctxt, ops, cs, VCPU_SREG_CS);
1600 if (rc != X86EMUL_CONTINUE)
1608 static int emulate_int(struct x86_emulate_ctxt *ctxt,
1609 struct x86_emulate_ops *ops, int irq)
1611 switch(ctxt->mode) {
1612 case X86EMUL_MODE_REAL:
1613 return emulate_int_real(ctxt, ops, irq);
1614 case X86EMUL_MODE_VM86:
1615 case X86EMUL_MODE_PROT16:
1616 case X86EMUL_MODE_PROT32:
1617 case X86EMUL_MODE_PROT64:
1619 /* Protected mode interrupts unimplemented yet */
1620 return X86EMUL_UNHANDLEABLE;
1624 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
1625 struct x86_emulate_ops *ops)
1627 struct decode_cache *c = &ctxt->decode;
1628 int rc = X86EMUL_CONTINUE;
1629 unsigned long temp_eip = 0;
1630 unsigned long temp_eflags = 0;
1631 unsigned long cs = 0;
1632 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1633 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1634 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1635 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1637 /* TODO: Add stack limit check */
1639 rc = emulate_pop(ctxt, ops, &temp_eip, c->op_bytes);
1641 if (rc != X86EMUL_CONTINUE)
1644 if (temp_eip & ~0xffff)
1645 return emulate_gp(ctxt, 0);
1647 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1649 if (rc != X86EMUL_CONTINUE)
1652 rc = emulate_pop(ctxt, ops, &temp_eflags, c->op_bytes);
1654 if (rc != X86EMUL_CONTINUE)
1657 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1659 if (rc != X86EMUL_CONTINUE)
1665 if (c->op_bytes == 4)
1666 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1667 else if (c->op_bytes == 2) {
1668 ctxt->eflags &= ~0xffff;
1669 ctxt->eflags |= temp_eflags;
1672 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1673 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1678 static inline int emulate_iret(struct x86_emulate_ctxt *ctxt,
1679 struct x86_emulate_ops* ops)
1681 switch(ctxt->mode) {
1682 case X86EMUL_MODE_REAL:
1683 return emulate_iret_real(ctxt, ops);
1684 case X86EMUL_MODE_VM86:
1685 case X86EMUL_MODE_PROT16:
1686 case X86EMUL_MODE_PROT32:
1687 case X86EMUL_MODE_PROT64:
1689 /* iret from protected mode unimplemented yet */
1690 return X86EMUL_UNHANDLEABLE;
1694 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1695 struct x86_emulate_ops *ops)
1697 struct decode_cache *c = &ctxt->decode;
1699 return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1702 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1704 struct decode_cache *c = &ctxt->decode;
1705 switch (c->modrm_reg) {
1707 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1710 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1713 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1716 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1718 case 4: /* sal/shl */
1719 case 6: /* sal/shl */
1720 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1723 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1726 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1731 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1732 struct x86_emulate_ops *ops)
1734 struct decode_cache *c = &ctxt->decode;
1735 unsigned long *rax = &c->regs[VCPU_REGS_RAX];
1736 unsigned long *rdx = &c->regs[VCPU_REGS_RDX];
1739 switch (c->modrm_reg) {
1740 case 0 ... 1: /* test */
1741 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1744 c->dst.val = ~c->dst.val;
1747 emulate_1op("neg", c->dst, ctxt->eflags);
1750 emulate_1op_rax_rdx("mul", c->src, *rax, *rdx, ctxt->eflags);
1753 emulate_1op_rax_rdx("imul", c->src, *rax, *rdx, ctxt->eflags);
1756 emulate_1op_rax_rdx_ex("div", c->src, *rax, *rdx,
1760 emulate_1op_rax_rdx_ex("idiv", c->src, *rax, *rdx,
1764 return X86EMUL_UNHANDLEABLE;
1767 return emulate_de(ctxt);
1768 return X86EMUL_CONTINUE;
1771 static int emulate_grp45(struct x86_emulate_ctxt *ctxt)
1773 struct decode_cache *c = &ctxt->decode;
1774 int rc = X86EMUL_CONTINUE;
1776 switch (c->modrm_reg) {
1778 emulate_1op("inc", c->dst, ctxt->eflags);
1781 emulate_1op("dec", c->dst, ctxt->eflags);
1783 case 2: /* call near abs */ {
1786 c->eip = c->src.val;
1787 c->src.val = old_eip;
1791 case 4: /* jmp abs */
1792 c->eip = c->src.val;
1801 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1802 struct x86_emulate_ops *ops)
1804 struct decode_cache *c = &ctxt->decode;
1805 u64 old = c->dst.orig_val64;
1807 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1808 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1809 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1810 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1811 ctxt->eflags &= ~EFLG_ZF;
1813 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1814 (u32) c->regs[VCPU_REGS_RBX];
1816 ctxt->eflags |= EFLG_ZF;
1818 return X86EMUL_CONTINUE;
1821 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1822 struct x86_emulate_ops *ops)
1824 struct decode_cache *c = &ctxt->decode;
1828 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1829 if (rc != X86EMUL_CONTINUE)
1831 if (c->op_bytes == 4)
1832 c->eip = (u32)c->eip;
1833 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1834 if (rc != X86EMUL_CONTINUE)
1836 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1840 static int emulate_load_segment(struct x86_emulate_ctxt *ctxt,
1841 struct x86_emulate_ops *ops, int seg)
1843 struct decode_cache *c = &ctxt->decode;
1847 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
1849 rc = load_segment_descriptor(ctxt, ops, sel, seg);
1850 if (rc != X86EMUL_CONTINUE)
1853 c->dst.val = c->src.val;
1858 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1859 struct x86_emulate_ops *ops, struct desc_struct *cs,
1860 struct desc_struct *ss)
1864 memset(cs, 0, sizeof(struct desc_struct));
1865 ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1866 memset(ss, 0, sizeof(struct desc_struct));
1868 cs->l = 0; /* will be adjusted later */
1869 set_desc_base(cs, 0); /* flat segment */
1870 cs->g = 1; /* 4kb granularity */
1871 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1872 cs->type = 0x0b; /* Read, Execute, Accessed */
1874 cs->dpl = 0; /* will be adjusted later */
1878 set_desc_base(ss, 0); /* flat segment */
1879 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1880 ss->g = 1; /* 4kb granularity */
1882 ss->type = 0x03; /* Read/Write, Accessed */
1883 ss->d = 1; /* 32bit stack segment */
1889 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1891 struct decode_cache *c = &ctxt->decode;
1892 struct desc_struct cs, ss;
1897 /* syscall is not available in real mode */
1898 if (ctxt->mode == X86EMUL_MODE_REAL ||
1899 ctxt->mode == X86EMUL_MODE_VM86)
1900 return emulate_ud(ctxt);
1902 ops->get_msr(ctxt, MSR_EFER, &efer);
1903 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1905 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1907 cs_sel = (u16)(msr_data & 0xfffc);
1908 ss_sel = (u16)(msr_data + 8);
1910 if (efer & EFER_LMA) {
1914 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1915 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1917 c->regs[VCPU_REGS_RCX] = c->eip;
1918 if (efer & EFER_LMA) {
1919 #ifdef CONFIG_X86_64
1920 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1923 ctxt->mode == X86EMUL_MODE_PROT64 ?
1924 MSR_LSTAR : MSR_CSTAR, &msr_data);
1927 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1928 ctxt->eflags &= ~(msr_data | EFLG_RF);
1932 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1933 c->eip = (u32)msr_data;
1935 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1938 return X86EMUL_CONTINUE;
1942 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1944 struct decode_cache *c = &ctxt->decode;
1945 struct desc_struct cs, ss;
1950 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1951 /* inject #GP if in real mode */
1952 if (ctxt->mode == X86EMUL_MODE_REAL)
1953 return emulate_gp(ctxt, 0);
1955 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1956 * Therefore, we inject an #UD.
1958 if (ctxt->mode == X86EMUL_MODE_PROT64)
1959 return emulate_ud(ctxt);
1961 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1963 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1964 switch (ctxt->mode) {
1965 case X86EMUL_MODE_PROT32:
1966 if ((msr_data & 0xfffc) == 0x0)
1967 return emulate_gp(ctxt, 0);
1969 case X86EMUL_MODE_PROT64:
1970 if (msr_data == 0x0)
1971 return emulate_gp(ctxt, 0);
1975 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1976 cs_sel = (u16)msr_data;
1977 cs_sel &= ~SELECTOR_RPL_MASK;
1978 ss_sel = cs_sel + 8;
1979 ss_sel &= ~SELECTOR_RPL_MASK;
1980 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
1985 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1986 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1988 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
1991 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
1992 c->regs[VCPU_REGS_RSP] = msr_data;
1994 return X86EMUL_CONTINUE;
1998 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2000 struct decode_cache *c = &ctxt->decode;
2001 struct desc_struct cs, ss;
2006 /* inject #GP if in real mode or Virtual 8086 mode */
2007 if (ctxt->mode == X86EMUL_MODE_REAL ||
2008 ctxt->mode == X86EMUL_MODE_VM86)
2009 return emulate_gp(ctxt, 0);
2011 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2013 if ((c->rex_prefix & 0x8) != 0x0)
2014 usermode = X86EMUL_MODE_PROT64;
2016 usermode = X86EMUL_MODE_PROT32;
2020 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2022 case X86EMUL_MODE_PROT32:
2023 cs_sel = (u16)(msr_data + 16);
2024 if ((msr_data & 0xfffc) == 0x0)
2025 return emulate_gp(ctxt, 0);
2026 ss_sel = (u16)(msr_data + 24);
2028 case X86EMUL_MODE_PROT64:
2029 cs_sel = (u16)(msr_data + 32);
2030 if (msr_data == 0x0)
2031 return emulate_gp(ctxt, 0);
2032 ss_sel = cs_sel + 8;
2037 cs_sel |= SELECTOR_RPL_MASK;
2038 ss_sel |= SELECTOR_RPL_MASK;
2040 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2041 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2043 c->eip = c->regs[VCPU_REGS_RDX];
2044 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
2046 return X86EMUL_CONTINUE;
2049 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
2050 struct x86_emulate_ops *ops)
2053 if (ctxt->mode == X86EMUL_MODE_REAL)
2055 if (ctxt->mode == X86EMUL_MODE_VM86)
2057 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2058 return ops->cpl(ctxt) > iopl;
2061 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2062 struct x86_emulate_ops *ops,
2065 struct desc_struct tr_seg;
2068 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2069 unsigned mask = (1 << len) - 1;
2072 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2075 if (desc_limit_scaled(&tr_seg) < 103)
2077 base = get_desc_base(&tr_seg);
2078 #ifdef CONFIG_X86_64
2079 base |= ((u64)base3) << 32;
2081 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2082 if (r != X86EMUL_CONTINUE)
2084 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2086 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2087 if (r != X86EMUL_CONTINUE)
2089 if ((perm >> bit_idx) & mask)
2094 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2095 struct x86_emulate_ops *ops,
2101 if (emulator_bad_iopl(ctxt, ops))
2102 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
2105 ctxt->perm_ok = true;
2110 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2111 struct x86_emulate_ops *ops,
2112 struct tss_segment_16 *tss)
2114 struct decode_cache *c = &ctxt->decode;
2117 tss->flag = ctxt->eflags;
2118 tss->ax = c->regs[VCPU_REGS_RAX];
2119 tss->cx = c->regs[VCPU_REGS_RCX];
2120 tss->dx = c->regs[VCPU_REGS_RDX];
2121 tss->bx = c->regs[VCPU_REGS_RBX];
2122 tss->sp = c->regs[VCPU_REGS_RSP];
2123 tss->bp = c->regs[VCPU_REGS_RBP];
2124 tss->si = c->regs[VCPU_REGS_RSI];
2125 tss->di = c->regs[VCPU_REGS_RDI];
2127 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2128 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2129 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2130 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2131 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2134 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2135 struct x86_emulate_ops *ops,
2136 struct tss_segment_16 *tss)
2138 struct decode_cache *c = &ctxt->decode;
2142 ctxt->eflags = tss->flag | 2;
2143 c->regs[VCPU_REGS_RAX] = tss->ax;
2144 c->regs[VCPU_REGS_RCX] = tss->cx;
2145 c->regs[VCPU_REGS_RDX] = tss->dx;
2146 c->regs[VCPU_REGS_RBX] = tss->bx;
2147 c->regs[VCPU_REGS_RSP] = tss->sp;
2148 c->regs[VCPU_REGS_RBP] = tss->bp;
2149 c->regs[VCPU_REGS_RSI] = tss->si;
2150 c->regs[VCPU_REGS_RDI] = tss->di;
2153 * SDM says that segment selectors are loaded before segment
2156 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2157 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2158 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2159 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2160 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2163 * Now load segment descriptors. If fault happenes at this stage
2164 * it is handled in a context of new task
2166 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
2167 if (ret != X86EMUL_CONTINUE)
2169 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2170 if (ret != X86EMUL_CONTINUE)
2172 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2173 if (ret != X86EMUL_CONTINUE)
2175 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2176 if (ret != X86EMUL_CONTINUE)
2178 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2179 if (ret != X86EMUL_CONTINUE)
2182 return X86EMUL_CONTINUE;
2185 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2186 struct x86_emulate_ops *ops,
2187 u16 tss_selector, u16 old_tss_sel,
2188 ulong old_tss_base, struct desc_struct *new_desc)
2190 struct tss_segment_16 tss_seg;
2192 u32 new_tss_base = get_desc_base(new_desc);
2194 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2196 if (ret != X86EMUL_CONTINUE)
2197 /* FIXME: need to provide precise fault address */
2200 save_state_to_tss16(ctxt, ops, &tss_seg);
2202 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2204 if (ret != X86EMUL_CONTINUE)
2205 /* FIXME: need to provide precise fault address */
2208 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2210 if (ret != X86EMUL_CONTINUE)
2211 /* FIXME: need to provide precise fault address */
2214 if (old_tss_sel != 0xffff) {
2215 tss_seg.prev_task_link = old_tss_sel;
2217 ret = ops->write_std(ctxt, new_tss_base,
2218 &tss_seg.prev_task_link,
2219 sizeof tss_seg.prev_task_link,
2221 if (ret != X86EMUL_CONTINUE)
2222 /* FIXME: need to provide precise fault address */
2226 return load_state_from_tss16(ctxt, ops, &tss_seg);
2229 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2230 struct x86_emulate_ops *ops,
2231 struct tss_segment_32 *tss)
2233 struct decode_cache *c = &ctxt->decode;
2235 tss->cr3 = ops->get_cr(ctxt, 3);
2237 tss->eflags = ctxt->eflags;
2238 tss->eax = c->regs[VCPU_REGS_RAX];
2239 tss->ecx = c->regs[VCPU_REGS_RCX];
2240 tss->edx = c->regs[VCPU_REGS_RDX];
2241 tss->ebx = c->regs[VCPU_REGS_RBX];
2242 tss->esp = c->regs[VCPU_REGS_RSP];
2243 tss->ebp = c->regs[VCPU_REGS_RBP];
2244 tss->esi = c->regs[VCPU_REGS_RSI];
2245 tss->edi = c->regs[VCPU_REGS_RDI];
2247 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2248 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2249 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2250 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2251 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2252 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2253 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2256 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2257 struct x86_emulate_ops *ops,
2258 struct tss_segment_32 *tss)
2260 struct decode_cache *c = &ctxt->decode;
2263 if (ops->set_cr(ctxt, 3, tss->cr3))
2264 return emulate_gp(ctxt, 0);
2266 ctxt->eflags = tss->eflags | 2;
2267 c->regs[VCPU_REGS_RAX] = tss->eax;
2268 c->regs[VCPU_REGS_RCX] = tss->ecx;
2269 c->regs[VCPU_REGS_RDX] = tss->edx;
2270 c->regs[VCPU_REGS_RBX] = tss->ebx;
2271 c->regs[VCPU_REGS_RSP] = tss->esp;
2272 c->regs[VCPU_REGS_RBP] = tss->ebp;
2273 c->regs[VCPU_REGS_RSI] = tss->esi;
2274 c->regs[VCPU_REGS_RDI] = tss->edi;
2277 * SDM says that segment selectors are loaded before segment
2280 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2281 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2282 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2283 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2284 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2285 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2286 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2289 * Now load segment descriptors. If fault happenes at this stage
2290 * it is handled in a context of new task
2292 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2293 if (ret != X86EMUL_CONTINUE)
2295 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2296 if (ret != X86EMUL_CONTINUE)
2298 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2299 if (ret != X86EMUL_CONTINUE)
2301 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2302 if (ret != X86EMUL_CONTINUE)
2304 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2305 if (ret != X86EMUL_CONTINUE)
2307 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2308 if (ret != X86EMUL_CONTINUE)
2310 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2311 if (ret != X86EMUL_CONTINUE)
2314 return X86EMUL_CONTINUE;
2317 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2318 struct x86_emulate_ops *ops,
2319 u16 tss_selector, u16 old_tss_sel,
2320 ulong old_tss_base, struct desc_struct *new_desc)
2322 struct tss_segment_32 tss_seg;
2324 u32 new_tss_base = get_desc_base(new_desc);
2326 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2328 if (ret != X86EMUL_CONTINUE)
2329 /* FIXME: need to provide precise fault address */
2332 save_state_to_tss32(ctxt, ops, &tss_seg);
2334 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2336 if (ret != X86EMUL_CONTINUE)
2337 /* FIXME: need to provide precise fault address */
2340 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2342 if (ret != X86EMUL_CONTINUE)
2343 /* FIXME: need to provide precise fault address */
2346 if (old_tss_sel != 0xffff) {
2347 tss_seg.prev_task_link = old_tss_sel;
2349 ret = ops->write_std(ctxt, new_tss_base,
2350 &tss_seg.prev_task_link,
2351 sizeof tss_seg.prev_task_link,
2353 if (ret != X86EMUL_CONTINUE)
2354 /* FIXME: need to provide precise fault address */
2358 return load_state_from_tss32(ctxt, ops, &tss_seg);
2361 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2362 struct x86_emulate_ops *ops,
2363 u16 tss_selector, int reason,
2364 bool has_error_code, u32 error_code)
2366 struct desc_struct curr_tss_desc, next_tss_desc;
2368 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2369 ulong old_tss_base =
2370 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2373 /* FIXME: old_tss_base == ~0 ? */
2375 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2376 if (ret != X86EMUL_CONTINUE)
2378 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2379 if (ret != X86EMUL_CONTINUE)
2382 /* FIXME: check that next_tss_desc is tss */
2384 if (reason != TASK_SWITCH_IRET) {
2385 if ((tss_selector & 3) > next_tss_desc.dpl ||
2386 ops->cpl(ctxt) > next_tss_desc.dpl)
2387 return emulate_gp(ctxt, 0);
2390 desc_limit = desc_limit_scaled(&next_tss_desc);
2391 if (!next_tss_desc.p ||
2392 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2393 desc_limit < 0x2b)) {
2394 emulate_ts(ctxt, tss_selector & 0xfffc);
2395 return X86EMUL_PROPAGATE_FAULT;
2398 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2399 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2400 write_segment_descriptor(ctxt, ops, old_tss_sel,
2404 if (reason == TASK_SWITCH_IRET)
2405 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2407 /* set back link to prev task only if NT bit is set in eflags
2408 note that old_tss_sel is not used afetr this point */
2409 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2410 old_tss_sel = 0xffff;
2412 if (next_tss_desc.type & 8)
2413 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2414 old_tss_base, &next_tss_desc);
2416 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2417 old_tss_base, &next_tss_desc);
2418 if (ret != X86EMUL_CONTINUE)
2421 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2422 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2424 if (reason != TASK_SWITCH_IRET) {
2425 next_tss_desc.type |= (1 << 1); /* set busy flag */
2426 write_segment_descriptor(ctxt, ops, tss_selector,
2430 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2431 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2433 if (has_error_code) {
2434 struct decode_cache *c = &ctxt->decode;
2436 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2438 c->src.val = (unsigned long) error_code;
2439 ret = em_push(ctxt);
2445 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2446 u16 tss_selector, int reason,
2447 bool has_error_code, u32 error_code)
2449 struct x86_emulate_ops *ops = ctxt->ops;
2450 struct decode_cache *c = &ctxt->decode;
2454 c->dst.type = OP_NONE;
2456 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2457 has_error_code, error_code);
2459 if (rc == X86EMUL_CONTINUE)
2462 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2465 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2466 int reg, struct operand *op)
2468 struct decode_cache *c = &ctxt->decode;
2469 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2471 register_address_increment(c, &c->regs[reg], df * op->bytes);
2472 op->addr.mem.ea = register_address(c, c->regs[reg]);
2473 op->addr.mem.seg = seg;
2476 static int em_das(struct x86_emulate_ctxt *ctxt)
2478 struct decode_cache *c = &ctxt->decode;
2480 bool af, cf, old_cf;
2482 cf = ctxt->eflags & X86_EFLAGS_CF;
2488 af = ctxt->eflags & X86_EFLAGS_AF;
2489 if ((al & 0x0f) > 9 || af) {
2491 cf = old_cf | (al >= 250);
2496 if (old_al > 0x99 || old_cf) {
2502 /* Set PF, ZF, SF */
2503 c->src.type = OP_IMM;
2506 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2507 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2509 ctxt->eflags |= X86_EFLAGS_CF;
2511 ctxt->eflags |= X86_EFLAGS_AF;
2512 return X86EMUL_CONTINUE;
2515 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2517 struct decode_cache *c = &ctxt->decode;
2522 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2525 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
2526 if (load_segment_descriptor(ctxt, ctxt->ops, sel, VCPU_SREG_CS))
2527 return X86EMUL_CONTINUE;
2530 memcpy(&c->eip, c->src.valptr, c->op_bytes);
2532 c->src.val = old_cs;
2534 if (rc != X86EMUL_CONTINUE)
2537 c->src.val = old_eip;
2538 return em_push(ctxt);
2541 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2543 struct decode_cache *c = &ctxt->decode;
2546 c->dst.type = OP_REG;
2547 c->dst.addr.reg = &c->eip;
2548 c->dst.bytes = c->op_bytes;
2549 rc = emulate_pop(ctxt, ctxt->ops, &c->dst.val, c->op_bytes);
2550 if (rc != X86EMUL_CONTINUE)
2552 register_address_increment(c, &c->regs[VCPU_REGS_RSP], c->src.val);
2553 return X86EMUL_CONTINUE;
2556 static int em_add(struct x86_emulate_ctxt *ctxt)
2558 struct decode_cache *c = &ctxt->decode;
2560 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
2561 return X86EMUL_CONTINUE;
2564 static int em_or(struct x86_emulate_ctxt *ctxt)
2566 struct decode_cache *c = &ctxt->decode;
2568 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2569 return X86EMUL_CONTINUE;
2572 static int em_adc(struct x86_emulate_ctxt *ctxt)
2574 struct decode_cache *c = &ctxt->decode;
2576 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
2577 return X86EMUL_CONTINUE;
2580 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2582 struct decode_cache *c = &ctxt->decode;
2584 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
2585 return X86EMUL_CONTINUE;
2588 static int em_and(struct x86_emulate_ctxt *ctxt)
2590 struct decode_cache *c = &ctxt->decode;
2592 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
2593 return X86EMUL_CONTINUE;
2596 static int em_sub(struct x86_emulate_ctxt *ctxt)
2598 struct decode_cache *c = &ctxt->decode;
2600 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
2601 return X86EMUL_CONTINUE;
2604 static int em_xor(struct x86_emulate_ctxt *ctxt)
2606 struct decode_cache *c = &ctxt->decode;
2608 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
2609 return X86EMUL_CONTINUE;
2612 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2614 struct decode_cache *c = &ctxt->decode;
2616 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2617 /* Disable writeback. */
2618 c->dst.type = OP_NONE;
2619 return X86EMUL_CONTINUE;
2622 static int em_imul(struct x86_emulate_ctxt *ctxt)
2624 struct decode_cache *c = &ctxt->decode;
2626 emulate_2op_SrcV_nobyte("imul", c->src, c->dst, ctxt->eflags);
2627 return X86EMUL_CONTINUE;
2630 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2632 struct decode_cache *c = &ctxt->decode;
2634 c->dst.val = c->src2.val;
2635 return em_imul(ctxt);
2638 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2640 struct decode_cache *c = &ctxt->decode;
2642 c->dst.type = OP_REG;
2643 c->dst.bytes = c->src.bytes;
2644 c->dst.addr.reg = &c->regs[VCPU_REGS_RDX];
2645 c->dst.val = ~((c->src.val >> (c->src.bytes * 8 - 1)) - 1);
2647 return X86EMUL_CONTINUE;
2650 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2652 struct decode_cache *c = &ctxt->decode;
2655 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2656 c->regs[VCPU_REGS_RAX] = (u32)tsc;
2657 c->regs[VCPU_REGS_RDX] = tsc >> 32;
2658 return X86EMUL_CONTINUE;
2661 static int em_mov(struct x86_emulate_ctxt *ctxt)
2663 struct decode_cache *c = &ctxt->decode;
2664 c->dst.val = c->src.val;
2665 return X86EMUL_CONTINUE;
2668 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2670 struct decode_cache *c = &ctxt->decode;
2671 memcpy(&c->dst.vec_val, &c->src.vec_val, c->op_bytes);
2672 return X86EMUL_CONTINUE;
2675 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2677 struct decode_cache *c = &ctxt->decode;
2681 rc = linearize(ctxt, c->src.addr.mem, 1, false, &linear);
2682 if (rc == X86EMUL_CONTINUE)
2683 ctxt->ops->invlpg(ctxt, linear);
2684 /* Disable writeback. */
2685 c->dst.type = OP_NONE;
2686 return X86EMUL_CONTINUE;
2689 static int em_clts(struct x86_emulate_ctxt *ctxt)
2693 cr0 = ctxt->ops->get_cr(ctxt, 0);
2695 ctxt->ops->set_cr(ctxt, 0, cr0);
2696 return X86EMUL_CONTINUE;
2699 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
2701 struct decode_cache *c = &ctxt->decode;
2704 if (c->modrm_mod != 3 || c->modrm_rm != 1)
2705 return X86EMUL_UNHANDLEABLE;
2707 rc = ctxt->ops->fix_hypercall(ctxt);
2708 if (rc != X86EMUL_CONTINUE)
2711 /* Let the processor re-execute the fixed hypercall */
2713 /* Disable writeback. */
2714 c->dst.type = OP_NONE;
2715 return X86EMUL_CONTINUE;
2718 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
2720 struct decode_cache *c = &ctxt->decode;
2721 struct desc_ptr desc_ptr;
2724 rc = read_descriptor(ctxt, ctxt->ops, c->src.addr.mem,
2725 &desc_ptr.size, &desc_ptr.address,
2727 if (rc != X86EMUL_CONTINUE)
2729 ctxt->ops->set_gdt(ctxt, &desc_ptr);
2730 /* Disable writeback. */
2731 c->dst.type = OP_NONE;
2732 return X86EMUL_CONTINUE;
2735 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2737 struct decode_cache *c = &ctxt->decode;
2740 rc = ctxt->ops->fix_hypercall(ctxt);
2742 /* Disable writeback. */
2743 c->dst.type = OP_NONE;
2747 static int em_lidt(struct x86_emulate_ctxt *ctxt)
2749 struct decode_cache *c = &ctxt->decode;
2750 struct desc_ptr desc_ptr;
2753 rc = read_descriptor(ctxt, ctxt->ops, c->src.addr.mem,
2757 if (rc != X86EMUL_CONTINUE)
2759 ctxt->ops->set_idt(ctxt, &desc_ptr);
2760 /* Disable writeback. */
2761 c->dst.type = OP_NONE;
2762 return X86EMUL_CONTINUE;
2765 static int em_smsw(struct x86_emulate_ctxt *ctxt)
2767 struct decode_cache *c = &ctxt->decode;
2770 c->dst.val = ctxt->ops->get_cr(ctxt, 0);
2771 return X86EMUL_CONTINUE;
2774 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
2776 struct decode_cache *c = &ctxt->decode;
2777 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2778 | (c->src.val & 0x0f));
2779 c->dst.type = OP_NONE;
2780 return X86EMUL_CONTINUE;
2783 static bool valid_cr(int nr)
2795 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2797 struct decode_cache *c = &ctxt->decode;
2799 if (!valid_cr(c->modrm_reg))
2800 return emulate_ud(ctxt);
2802 return X86EMUL_CONTINUE;
2805 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2807 struct decode_cache *c = &ctxt->decode;
2808 u64 new_val = c->src.val64;
2809 int cr = c->modrm_reg;
2812 static u64 cr_reserved_bits[] = {
2813 0xffffffff00000000ULL,
2814 0, 0, 0, /* CR3 checked later */
2821 return emulate_ud(ctxt);
2823 if (new_val & cr_reserved_bits[cr])
2824 return emulate_gp(ctxt, 0);
2829 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2830 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2831 return emulate_gp(ctxt, 0);
2833 cr4 = ctxt->ops->get_cr(ctxt, 4);
2834 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2836 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2837 !(cr4 & X86_CR4_PAE))
2838 return emulate_gp(ctxt, 0);
2845 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2846 if (efer & EFER_LMA)
2847 rsvd = CR3_L_MODE_RESERVED_BITS;
2848 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
2849 rsvd = CR3_PAE_RESERVED_BITS;
2850 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
2851 rsvd = CR3_NONPAE_RESERVED_BITS;
2854 return emulate_gp(ctxt, 0);
2861 cr4 = ctxt->ops->get_cr(ctxt, 4);
2862 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2864 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2865 return emulate_gp(ctxt, 0);
2871 return X86EMUL_CONTINUE;
2874 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2878 ctxt->ops->get_dr(ctxt, 7, &dr7);
2880 /* Check if DR7.Global_Enable is set */
2881 return dr7 & (1 << 13);
2884 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2886 struct decode_cache *c = &ctxt->decode;
2887 int dr = c->modrm_reg;
2891 return emulate_ud(ctxt);
2893 cr4 = ctxt->ops->get_cr(ctxt, 4);
2894 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2895 return emulate_ud(ctxt);
2897 if (check_dr7_gd(ctxt))
2898 return emulate_db(ctxt);
2900 return X86EMUL_CONTINUE;
2903 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2905 struct decode_cache *c = &ctxt->decode;
2906 u64 new_val = c->src.val64;
2907 int dr = c->modrm_reg;
2909 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2910 return emulate_gp(ctxt, 0);
2912 return check_dr_read(ctxt);
2915 static int check_svme(struct x86_emulate_ctxt *ctxt)
2919 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2921 if (!(efer & EFER_SVME))
2922 return emulate_ud(ctxt);
2924 return X86EMUL_CONTINUE;
2927 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
2929 u64 rax = ctxt->decode.regs[VCPU_REGS_RAX];
2931 /* Valid physical address? */
2932 if (rax & 0xffff000000000000ULL)
2933 return emulate_gp(ctxt, 0);
2935 return check_svme(ctxt);
2938 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
2940 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2942 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
2943 return emulate_ud(ctxt);
2945 return X86EMUL_CONTINUE;
2948 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
2950 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2951 u64 rcx = ctxt->decode.regs[VCPU_REGS_RCX];
2953 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
2955 return emulate_gp(ctxt, 0);
2957 return X86EMUL_CONTINUE;
2960 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
2962 struct decode_cache *c = &ctxt->decode;
2964 c->dst.bytes = min(c->dst.bytes, 4u);
2965 if (!emulator_io_permited(ctxt, ctxt->ops, c->src.val, c->dst.bytes))
2966 return emulate_gp(ctxt, 0);
2968 return X86EMUL_CONTINUE;
2971 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
2973 struct decode_cache *c = &ctxt->decode;
2975 c->src.bytes = min(c->src.bytes, 4u);
2976 if (!emulator_io_permited(ctxt, ctxt->ops, c->dst.val, c->src.bytes))
2977 return emulate_gp(ctxt, 0);
2979 return X86EMUL_CONTINUE;
2982 #define D(_y) { .flags = (_y) }
2983 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2984 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
2985 .check_perm = (_p) }
2987 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
2988 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2989 #define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
2990 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2991 #define II(_f, _e, _i) \
2992 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2993 #define IIP(_f, _e, _i, _p) \
2994 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
2995 .check_perm = (_p) }
2996 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
2998 #define D2bv(_f) D((_f) | ByteOp), D(_f)
2999 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3000 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3002 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3003 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3004 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3006 static struct opcode group7_rm1[] = {
3007 DI(SrcNone | ModRM | Priv, monitor),
3008 DI(SrcNone | ModRM | Priv, mwait),
3012 static struct opcode group7_rm3[] = {
3013 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
3014 II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
3015 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
3016 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
3017 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
3018 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
3019 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
3020 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
3023 static struct opcode group7_rm7[] = {
3025 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
3029 static struct opcode group1[] = {
3040 static struct opcode group1A[] = {
3041 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
3044 static struct opcode group3[] = {
3045 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
3046 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3047 X4(D(SrcMem | ModRM)),
3050 static struct opcode group4[] = {
3051 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
3055 static struct opcode group5[] = {
3056 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3057 D(SrcMem | ModRM | Stack),
3058 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3059 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
3060 D(SrcMem | ModRM | Stack), N,
3063 static struct opcode group6[] = {
3064 DI(ModRM | Prot, sldt),
3065 DI(ModRM | Prot, str),
3066 DI(ModRM | Prot | Priv, lldt),
3067 DI(ModRM | Prot | Priv, ltr),
3071 static struct group_dual group7 = { {
3072 DI(ModRM | Mov | DstMem | Priv, sgdt),
3073 DI(ModRM | Mov | DstMem | Priv, sidt),
3074 II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
3075 II(ModRM | SrcMem | Priv, em_lidt, lidt),
3076 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3077 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
3078 II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3080 I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
3082 N, EXT(0, group7_rm3),
3083 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3084 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3087 static struct opcode group8[] = {
3089 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
3090 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
3093 static struct group_dual group9 = { {
3094 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
3096 N, N, N, N, N, N, N, N,
3099 static struct opcode group11[] = {
3100 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
3103 static struct gprefix pfx_0f_6f_0f_7f = {
3104 N, N, N, I(Sse, em_movdqu),
3107 static struct opcode opcode_table[256] = {
3109 I6ALU(Lock, em_add),
3110 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3113 D(ImplicitOps | Stack | No64), N,
3115 I6ALU(Lock, em_adc),
3116 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3118 I6ALU(Lock, em_sbb),
3119 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3121 I6ALU(Lock, em_and), N, N,
3123 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3125 I6ALU(Lock, em_xor), N, N,
3127 I6ALU(0, em_cmp), N, N,
3131 X8(I(SrcReg | Stack, em_push)),
3133 X8(I(DstReg | Stack, em_pop)),
3135 I(ImplicitOps | Stack | No64, em_pusha),
3136 I(ImplicitOps | Stack | No64, em_popa),
3137 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3140 I(SrcImm | Mov | Stack, em_push),
3141 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3142 I(SrcImmByte | Mov | Stack, em_push),
3143 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3144 D2bvIP(DstDI | Mov | String, ins, check_perm_in), /* insb, insw/insd */
3145 D2bvIP(SrcSI | ImplicitOps | String, outs, check_perm_out), /* outsb, outsw/outsd */
3149 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
3150 G(DstMem | SrcImm | ModRM | Group, group1),
3151 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
3152 G(DstMem | SrcImmByte | ModRM | Group, group1),
3153 D2bv(DstMem | SrcReg | ModRM), D2bv(DstMem | SrcReg | ModRM | Lock),
3155 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
3156 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3157 D(DstMem | SrcNone | ModRM | Mov), D(ModRM | SrcMem | NoAccess | DstReg),
3158 D(ImplicitOps | SrcMem16 | ModRM), G(0, group1A),
3160 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3162 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3163 I(SrcImmFAddr | No64, em_call_far), N,
3164 II(ImplicitOps | Stack, em_pushf, pushf),
3165 II(ImplicitOps | Stack, em_popf, popf), N, N,
3167 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3168 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
3169 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3170 I2bv(SrcSI | DstDI | String, em_cmp),
3172 D2bv(DstAcc | SrcImm),
3173 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3174 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3175 I2bv(SrcAcc | DstDI | String, em_cmp),
3177 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3179 X8(I(DstReg | SrcImm | Mov, em_mov)),
3181 D2bv(DstMem | SrcImmByte | ModRM),
3182 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3183 D(ImplicitOps | Stack),
3184 D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
3185 G(ByteOp, group11), G(0, group11),
3187 N, N, N, D(ImplicitOps | Stack),
3188 D(ImplicitOps), DI(SrcImmByte, intn),
3189 D(ImplicitOps | No64), DI(ImplicitOps, iret),
3191 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3194 N, N, N, N, N, N, N, N,
3197 D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
3198 D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
3200 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
3201 D(SrcImmFAddr | No64), D(SrcImmByte | ImplicitOps),
3202 D2bvIP(SrcNone | DstAcc, in, check_perm_in),
3203 D2bvIP(SrcAcc | ImplicitOps, out, check_perm_out),
3205 N, DI(ImplicitOps, icebp), N, N,
3206 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3207 G(ByteOp, group3), G(0, group3),
3209 D(ImplicitOps), D(ImplicitOps), D(ImplicitOps), D(ImplicitOps),
3210 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3213 static struct opcode twobyte_table[256] = {
3215 G(0, group6), GD(0, &group7), N, N,
3216 N, D(ImplicitOps | VendorSpecific), DI(ImplicitOps | Priv, clts), N,
3217 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3218 N, D(ImplicitOps | ModRM), N, N,
3220 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3222 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3223 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3224 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3225 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3227 N, N, N, N, N, N, N, N,
3229 DI(ImplicitOps | Priv, wrmsr),
3230 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3231 DI(ImplicitOps | Priv, rdmsr),
3232 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3233 D(ImplicitOps | VendorSpecific), D(ImplicitOps | Priv | VendorSpecific),
3235 N, N, N, N, N, N, N, N,
3237 X16(D(DstReg | SrcMem | ModRM | Mov)),
3239 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3244 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3249 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3253 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3255 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3256 DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3257 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3258 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3260 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3261 DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3262 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3263 D(DstMem | SrcReg | Src2CL | ModRM),
3264 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3266 D2bv(DstMem | SrcReg | ModRM | Lock),
3267 D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3268 D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
3269 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3272 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3273 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3274 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3276 D2bv(DstMem | SrcReg | ModRM | Lock),
3277 N, D(DstMem | SrcReg | ModRM | Mov),
3278 N, N, N, GD(0, &group9),
3279 N, N, N, N, N, N, N, N,
3281 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3283 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3285 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3301 static unsigned imm_size(struct decode_cache *c)
3305 size = (c->d & ByteOp) ? 1 : c->op_bytes;
3311 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3312 unsigned size, bool sign_extension)
3314 struct decode_cache *c = &ctxt->decode;
3315 struct x86_emulate_ops *ops = ctxt->ops;
3316 int rc = X86EMUL_CONTINUE;
3320 op->addr.mem.ea = c->eip;
3321 /* NB. Immediates are sign-extended as necessary. */
3322 switch (op->bytes) {
3324 op->val = insn_fetch(s8, 1, c->eip);
3327 op->val = insn_fetch(s16, 2, c->eip);
3330 op->val = insn_fetch(s32, 4, c->eip);
3333 if (!sign_extension) {
3334 switch (op->bytes) {
3342 op->val &= 0xffffffff;
3351 x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3353 struct x86_emulate_ops *ops = ctxt->ops;
3354 struct decode_cache *c = &ctxt->decode;
3355 int rc = X86EMUL_CONTINUE;
3356 int mode = ctxt->mode;
3357 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3358 bool op_prefix = false;
3359 struct opcode opcode;
3360 struct operand memop = { .type = OP_NONE };
3363 c->fetch.start = c->eip;
3364 c->fetch.end = c->fetch.start + insn_len;
3366 memcpy(c->fetch.data, insn, insn_len);
3369 case X86EMUL_MODE_REAL:
3370 case X86EMUL_MODE_VM86:
3371 case X86EMUL_MODE_PROT16:
3372 def_op_bytes = def_ad_bytes = 2;
3374 case X86EMUL_MODE_PROT32:
3375 def_op_bytes = def_ad_bytes = 4;
3377 #ifdef CONFIG_X86_64
3378 case X86EMUL_MODE_PROT64:
3387 c->op_bytes = def_op_bytes;
3388 c->ad_bytes = def_ad_bytes;
3390 /* Legacy prefixes. */
3392 switch (c->b = insn_fetch(u8, 1, c->eip)) {
3393 case 0x66: /* operand-size override */
3395 /* switch between 2/4 bytes */
3396 c->op_bytes = def_op_bytes ^ 6;
3398 case 0x67: /* address-size override */
3399 if (mode == X86EMUL_MODE_PROT64)
3400 /* switch between 4/8 bytes */
3401 c->ad_bytes = def_ad_bytes ^ 12;
3403 /* switch between 2/4 bytes */
3404 c->ad_bytes = def_ad_bytes ^ 6;
3406 case 0x26: /* ES override */
3407 case 0x2e: /* CS override */
3408 case 0x36: /* SS override */
3409 case 0x3e: /* DS override */
3410 set_seg_override(c, (c->b >> 3) & 3);
3412 case 0x64: /* FS override */
3413 case 0x65: /* GS override */
3414 set_seg_override(c, c->b & 7);
3416 case 0x40 ... 0x4f: /* REX */
3417 if (mode != X86EMUL_MODE_PROT64)
3419 c->rex_prefix = c->b;
3421 case 0xf0: /* LOCK */
3424 case 0xf2: /* REPNE/REPNZ */
3425 case 0xf3: /* REP/REPE/REPZ */
3426 c->rep_prefix = c->b;
3432 /* Any legacy prefix after a REX prefix nullifies its effect. */
3440 if (c->rex_prefix & 8)
3441 c->op_bytes = 8; /* REX.W */
3443 /* Opcode byte(s). */
3444 opcode = opcode_table[c->b];
3445 /* Two-byte opcode? */
3448 c->b = insn_fetch(u8, 1, c->eip);
3449 opcode = twobyte_table[c->b];
3451 c->d = opcode.flags;
3453 while (c->d & GroupMask) {
3454 switch (c->d & GroupMask) {
3456 c->modrm = insn_fetch(u8, 1, c->eip);
3458 goffset = (c->modrm >> 3) & 7;
3459 opcode = opcode.u.group[goffset];
3462 c->modrm = insn_fetch(u8, 1, c->eip);
3464 goffset = (c->modrm >> 3) & 7;
3465 if ((c->modrm >> 6) == 3)
3466 opcode = opcode.u.gdual->mod3[goffset];
3468 opcode = opcode.u.gdual->mod012[goffset];
3471 goffset = c->modrm & 7;
3472 opcode = opcode.u.group[goffset];
3475 if (c->rep_prefix && op_prefix)
3476 return X86EMUL_UNHANDLEABLE;
3477 simd_prefix = op_prefix ? 0x66 : c->rep_prefix;
3478 switch (simd_prefix) {
3479 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3480 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3481 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3482 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3486 return X86EMUL_UNHANDLEABLE;
3490 c->d |= opcode.flags;
3493 c->execute = opcode.u.execute;
3494 c->check_perm = opcode.check_perm;
3495 c->intercept = opcode.intercept;
3498 if (c->d == 0 || (c->d & Undefined))
3501 if (!(c->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3504 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
3507 if (c->d & Op3264) {
3508 if (mode == X86EMUL_MODE_PROT64)
3517 /* ModRM and SIB bytes. */
3519 rc = decode_modrm(ctxt, ops, &memop);
3520 if (!c->has_seg_override)
3521 set_seg_override(c, c->modrm_seg);
3522 } else if (c->d & MemAbs)
3523 rc = decode_abs(ctxt, ops, &memop);
3524 if (rc != X86EMUL_CONTINUE)
3527 if (!c->has_seg_override)
3528 set_seg_override(c, VCPU_SREG_DS);
3530 memop.addr.mem.seg = seg_override(ctxt, ops, c);
3532 if (memop.type == OP_MEM && c->ad_bytes != 8)
3533 memop.addr.mem.ea = (u32)memop.addr.mem.ea;
3535 if (memop.type == OP_MEM && c->rip_relative)
3536 memop.addr.mem.ea += c->eip;
3539 * Decode and fetch the source operand: register, memory
3542 switch (c->d & SrcMask) {
3546 decode_register_operand(ctxt, &c->src, c, 0);
3555 memop.bytes = (c->d & ByteOp) ? 1 :
3561 rc = decode_imm(ctxt, &c->src, 2, false);
3564 rc = decode_imm(ctxt, &c->src, imm_size(c), true);
3567 rc = decode_imm(ctxt, &c->src, imm_size(c), false);
3570 rc = decode_imm(ctxt, &c->src, 1, true);
3573 rc = decode_imm(ctxt, &c->src, 1, false);
3576 c->src.type = OP_REG;
3577 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3578 c->src.addr.reg = &c->regs[VCPU_REGS_RAX];
3579 fetch_register_operand(&c->src);
3586 c->src.type = OP_MEM;
3587 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3588 c->src.addr.mem.ea =
3589 register_address(c, c->regs[VCPU_REGS_RSI]);
3590 c->src.addr.mem.seg = seg_override(ctxt, ops, c),
3594 c->src.type = OP_IMM;
3595 c->src.addr.mem.ea = c->eip;
3596 c->src.bytes = c->op_bytes + 2;
3597 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
3600 memop.bytes = c->op_bytes + 2;
3605 if (rc != X86EMUL_CONTINUE)
3609 * Decode and fetch the second source operand: register, memory
3612 switch (c->d & Src2Mask) {
3617 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
3620 rc = decode_imm(ctxt, &c->src2, 1, true);
3627 rc = decode_imm(ctxt, &c->src2, imm_size(c), true);
3631 if (rc != X86EMUL_CONTINUE)
3634 /* Decode and fetch the destination operand: register or memory. */
3635 switch (c->d & DstMask) {
3637 decode_register_operand(ctxt, &c->dst, c,
3638 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
3641 c->dst.type = OP_IMM;
3642 c->dst.addr.mem.ea = c->eip;
3644 c->dst.val = insn_fetch(u8, 1, c->eip);
3649 if ((c->d & DstMask) == DstMem64)
3652 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3654 fetch_bit_operand(c);
3655 c->dst.orig_val = c->dst.val;
3658 c->dst.type = OP_REG;
3659 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3660 c->dst.addr.reg = &c->regs[VCPU_REGS_RAX];
3661 fetch_register_operand(&c->dst);
3662 c->dst.orig_val = c->dst.val;
3665 c->dst.type = OP_MEM;
3666 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3667 c->dst.addr.mem.ea =
3668 register_address(c, c->regs[VCPU_REGS_RDI]);
3669 c->dst.addr.mem.seg = VCPU_SREG_ES;
3673 /* Special instructions do their own operand decoding. */
3675 c->dst.type = OP_NONE; /* Disable writeback. */
3680 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3683 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3685 struct decode_cache *c = &ctxt->decode;
3687 /* The second termination condition only applies for REPE
3688 * and REPNE. Test if the repeat string operation prefix is
3689 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3690 * corresponding termination condition according to:
3691 * - if REPE/REPZ and ZF = 0 then done
3692 * - if REPNE/REPNZ and ZF = 1 then done
3694 if (((c->b == 0xa6) || (c->b == 0xa7) ||
3695 (c->b == 0xae) || (c->b == 0xaf))
3696 && (((c->rep_prefix == REPE_PREFIX) &&
3697 ((ctxt->eflags & EFLG_ZF) == 0))
3698 || ((c->rep_prefix == REPNE_PREFIX) &&
3699 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3706 x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3708 struct x86_emulate_ops *ops = ctxt->ops;
3710 struct decode_cache *c = &ctxt->decode;
3711 int rc = X86EMUL_CONTINUE;
3712 int saved_dst_type = c->dst.type;
3713 int irq; /* Used for int 3, int, and into */
3715 ctxt->decode.mem_read.pos = 0;
3717 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
3718 rc = emulate_ud(ctxt);
3722 /* LOCK prefix is allowed only with some instructions */
3723 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
3724 rc = emulate_ud(ctxt);
3728 if ((c->d & SrcMask) == SrcMemFAddr && c->src.type != OP_MEM) {
3729 rc = emulate_ud(ctxt);
3734 && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
3735 || !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
3736 rc = emulate_ud(ctxt);
3740 if ((c->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
3741 rc = emulate_nm(ctxt);
3745 if (unlikely(ctxt->guest_mode) && c->intercept) {
3746 rc = emulator_check_intercept(ctxt, c->intercept,
3747 X86_ICPT_PRE_EXCEPT);
3748 if (rc != X86EMUL_CONTINUE)
3752 /* Privileged instruction can be executed only in CPL=0 */
3753 if ((c->d & Priv) && ops->cpl(ctxt)) {
3754 rc = emulate_gp(ctxt, 0);
3758 /* Instruction can only be executed in protected mode */
3759 if ((c->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3760 rc = emulate_ud(ctxt);
3764 /* Do instruction specific permission checks */
3765 if (c->check_perm) {
3766 rc = c->check_perm(ctxt);
3767 if (rc != X86EMUL_CONTINUE)
3771 if (unlikely(ctxt->guest_mode) && c->intercept) {
3772 rc = emulator_check_intercept(ctxt, c->intercept,
3773 X86_ICPT_POST_EXCEPT);
3774 if (rc != X86EMUL_CONTINUE)
3778 if (c->rep_prefix && (c->d & String)) {
3779 /* All REP prefixes have the same first termination condition */
3780 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
3786 if ((c->src.type == OP_MEM) && !(c->d & NoAccess)) {
3787 rc = segmented_read(ctxt, c->src.addr.mem,
3788 c->src.valptr, c->src.bytes);
3789 if (rc != X86EMUL_CONTINUE)
3791 c->src.orig_val64 = c->src.val64;
3794 if (c->src2.type == OP_MEM) {
3795 rc = segmented_read(ctxt, c->src2.addr.mem,
3796 &c->src2.val, c->src2.bytes);
3797 if (rc != X86EMUL_CONTINUE)
3801 if ((c->d & DstMask) == ImplicitOps)
3805 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
3806 /* optimisation - avoid slow emulated read if Mov */
3807 rc = segmented_read(ctxt, c->dst.addr.mem,
3808 &c->dst.val, c->dst.bytes);
3809 if (rc != X86EMUL_CONTINUE)
3812 c->dst.orig_val = c->dst.val;
3816 if (unlikely(ctxt->guest_mode) && c->intercept) {
3817 rc = emulator_check_intercept(ctxt, c->intercept,
3818 X86_ICPT_POST_MEMACCESS);
3819 if (rc != X86EMUL_CONTINUE)
3824 rc = c->execute(ctxt);
3825 if (rc != X86EMUL_CONTINUE)
3834 case 0x06: /* push es */
3835 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
3837 case 0x07: /* pop es */
3838 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
3840 case 0x0e: /* push cs */
3841 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
3843 case 0x16: /* push ss */
3844 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
3846 case 0x17: /* pop ss */
3847 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
3849 case 0x1e: /* push ds */
3850 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
3852 case 0x1f: /* pop ds */
3853 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
3855 case 0x40 ... 0x47: /* inc r16/r32 */
3856 emulate_1op("inc", c->dst, ctxt->eflags);
3858 case 0x48 ... 0x4f: /* dec r16/r32 */
3859 emulate_1op("dec", c->dst, ctxt->eflags);
3861 case 0x63: /* movsxd */
3862 if (ctxt->mode != X86EMUL_MODE_PROT64)
3863 goto cannot_emulate;
3864 c->dst.val = (s32) c->src.val;
3866 case 0x6c: /* insb */
3867 case 0x6d: /* insw/insd */
3868 c->src.val = c->regs[VCPU_REGS_RDX];
3870 case 0x6e: /* outsb */
3871 case 0x6f: /* outsw/outsd */
3872 c->dst.val = c->regs[VCPU_REGS_RDX];
3875 case 0x70 ... 0x7f: /* jcc (short) */
3876 if (test_cc(c->b, ctxt->eflags))
3877 jmp_rel(c, c->src.val);
3881 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
3883 case 0x86 ... 0x87: /* xchg */
3885 /* Write back the register source. */
3886 c->src.val = c->dst.val;
3887 write_register_operand(&c->src);
3889 * Write back the memory destination with implicit LOCK
3892 c->dst.val = c->src.orig_val;
3895 case 0x8c: /* mov r/m, sreg */
3896 if (c->modrm_reg > VCPU_SREG_GS) {
3897 rc = emulate_ud(ctxt);
3900 c->dst.val = get_segment_selector(ctxt, c->modrm_reg);
3902 case 0x8d: /* lea r16/r32, m */
3903 c->dst.val = c->src.addr.mem.ea;
3905 case 0x8e: { /* mov seg, r/m16 */
3910 if (c->modrm_reg == VCPU_SREG_CS ||
3911 c->modrm_reg > VCPU_SREG_GS) {
3912 rc = emulate_ud(ctxt);
3916 if (c->modrm_reg == VCPU_SREG_SS)
3917 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3919 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
3921 c->dst.type = OP_NONE; /* Disable writeback. */
3924 case 0x8f: /* pop (sole member of Grp1a) */
3925 rc = emulate_grp1a(ctxt, ops);
3927 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3928 if (c->dst.addr.reg == &c->regs[VCPU_REGS_RAX])
3931 case 0x98: /* cbw/cwde/cdqe */
3932 switch (c->op_bytes) {
3933 case 2: c->dst.val = (s8)c->dst.val; break;
3934 case 4: c->dst.val = (s16)c->dst.val; break;
3935 case 8: c->dst.val = (s32)c->dst.val; break;
3938 case 0xa8 ... 0xa9: /* test ax, imm */
3943 case 0xc3: /* ret */
3944 c->dst.type = OP_REG;
3945 c->dst.addr.reg = &c->eip;
3946 c->dst.bytes = c->op_bytes;
3949 case 0xc4: /* les */
3950 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_ES);
3952 case 0xc5: /* lds */
3953 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_DS);
3955 case 0xcb: /* ret far */
3956 rc = emulate_ret_far(ctxt, ops);
3958 case 0xcc: /* int3 */
3961 case 0xcd: /* int n */
3964 rc = emulate_int(ctxt, ops, irq);
3966 case 0xce: /* into */
3967 if (ctxt->eflags & EFLG_OF) {
3972 case 0xcf: /* iret */
3973 rc = emulate_iret(ctxt, ops);
3975 case 0xd0 ... 0xd1: /* Grp2 */
3978 case 0xd2 ... 0xd3: /* Grp2 */
3979 c->src.val = c->regs[VCPU_REGS_RCX];
3982 case 0xe0 ... 0xe2: /* loop/loopz/loopnz */
3983 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3984 if (address_mask(c, c->regs[VCPU_REGS_RCX]) != 0 &&
3985 (c->b == 0xe2 || test_cc(c->b ^ 0x5, ctxt->eflags)))
3986 jmp_rel(c, c->src.val);
3988 case 0xe3: /* jcxz/jecxz/jrcxz */
3989 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0)
3990 jmp_rel(c, c->src.val);
3992 case 0xe4: /* inb */
3995 case 0xe6: /* outb */
3996 case 0xe7: /* out */
3998 case 0xe8: /* call (near) */ {
3999 long int rel = c->src.val;
4000 c->src.val = (unsigned long) c->eip;
4005 case 0xe9: /* jmp rel */
4007 case 0xea: { /* jmp far */
4010 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
4012 rc = load_segment_descriptor(ctxt, ops, sel, VCPU_SREG_CS);
4013 if (rc != X86EMUL_CONTINUE)
4017 memcpy(&c->eip, c->src.valptr, c->op_bytes);
4021 jmp: /* jmp rel short */
4022 jmp_rel(c, c->src.val);
4023 c->dst.type = OP_NONE; /* Disable writeback. */
4025 case 0xec: /* in al,dx */
4026 case 0xed: /* in (e/r)ax,dx */
4027 c->src.val = c->regs[VCPU_REGS_RDX];
4029 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
4031 goto done; /* IO is needed */
4033 case 0xee: /* out dx,al */
4034 case 0xef: /* out dx,(e/r)ax */
4035 c->dst.val = c->regs[VCPU_REGS_RDX];
4037 ops->pio_out_emulated(ctxt, c->src.bytes, c->dst.val,
4039 c->dst.type = OP_NONE; /* Disable writeback. */
4041 case 0xf4: /* hlt */
4042 ctxt->ops->halt(ctxt);
4044 case 0xf5: /* cmc */
4045 /* complement carry flag from eflags reg */
4046 ctxt->eflags ^= EFLG_CF;
4048 case 0xf6 ... 0xf7: /* Grp3 */
4049 rc = emulate_grp3(ctxt, ops);
4051 case 0xf8: /* clc */
4052 ctxt->eflags &= ~EFLG_CF;
4054 case 0xf9: /* stc */
4055 ctxt->eflags |= EFLG_CF;
4057 case 0xfa: /* cli */
4058 if (emulator_bad_iopl(ctxt, ops)) {
4059 rc = emulate_gp(ctxt, 0);
4062 ctxt->eflags &= ~X86_EFLAGS_IF;
4064 case 0xfb: /* sti */
4065 if (emulator_bad_iopl(ctxt, ops)) {
4066 rc = emulate_gp(ctxt, 0);
4069 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
4070 ctxt->eflags |= X86_EFLAGS_IF;
4073 case 0xfc: /* cld */
4074 ctxt->eflags &= ~EFLG_DF;
4076 case 0xfd: /* std */
4077 ctxt->eflags |= EFLG_DF;
4079 case 0xfe: /* Grp4 */
4081 rc = emulate_grp45(ctxt);
4083 case 0xff: /* Grp5 */
4084 if (c->modrm_reg == 5)
4088 goto cannot_emulate;
4091 if (rc != X86EMUL_CONTINUE)
4095 rc = writeback(ctxt, ops);
4096 if (rc != X86EMUL_CONTINUE)
4100 * restore dst type in case the decoding will be reused
4101 * (happens for string instruction )
4103 c->dst.type = saved_dst_type;
4105 if ((c->d & SrcMask) == SrcSI)
4106 string_addr_inc(ctxt, seg_override(ctxt, ops, c),
4107 VCPU_REGS_RSI, &c->src);
4109 if ((c->d & DstMask) == DstDI)
4110 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4113 if (c->rep_prefix && (c->d & String)) {
4114 struct read_cache *r = &ctxt->decode.io_read;
4115 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
4117 if (!string_insn_completed(ctxt)) {
4119 * Re-enter guest when pio read ahead buffer is empty
4120 * or, if it is not used, after each 1024 iteration.
4122 if ((r->end != 0 || c->regs[VCPU_REGS_RCX] & 0x3ff) &&
4123 (r->end == 0 || r->end != r->pos)) {
4125 * Reset read cache. Usually happens before
4126 * decode, but since instruction is restarted
4127 * we have to do it here.
4129 ctxt->decode.mem_read.end = 0;
4130 return EMULATION_RESTART;
4132 goto done; /* skip rip writeback */
4139 if (rc == X86EMUL_PROPAGATE_FAULT)
4140 ctxt->have_exception = true;
4141 if (rc == X86EMUL_INTERCEPTED)
4142 return EMULATION_INTERCEPTED;
4144 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4148 case 0x05: /* syscall */
4149 rc = emulate_syscall(ctxt, ops);
4154 case 0x09: /* wbinvd */
4155 (ctxt->ops->wbinvd)(ctxt);
4157 case 0x08: /* invd */
4158 case 0x0d: /* GrpP (prefetch) */
4159 case 0x18: /* Grp16 (prefetch/nop) */
4161 case 0x20: /* mov cr, reg */
4162 c->dst.val = ops->get_cr(ctxt, c->modrm_reg);
4164 case 0x21: /* mov from dr to reg */
4165 ops->get_dr(ctxt, c->modrm_reg, &c->dst.val);
4167 case 0x22: /* mov reg, cr */
4168 if (ops->set_cr(ctxt, c->modrm_reg, c->src.val)) {
4169 emulate_gp(ctxt, 0);
4170 rc = X86EMUL_PROPAGATE_FAULT;
4173 c->dst.type = OP_NONE;
4175 case 0x23: /* mov from reg to dr */
4176 if (ops->set_dr(ctxt, c->modrm_reg, c->src.val &
4177 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4178 ~0ULL : ~0U)) < 0) {
4179 /* #UD condition is already handled by the code above */
4180 emulate_gp(ctxt, 0);
4181 rc = X86EMUL_PROPAGATE_FAULT;
4185 c->dst.type = OP_NONE; /* no writeback */
4189 msr_data = (u32)c->regs[VCPU_REGS_RAX]
4190 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
4191 if (ops->set_msr(ctxt, c->regs[VCPU_REGS_RCX], msr_data)) {
4192 emulate_gp(ctxt, 0);
4193 rc = X86EMUL_PROPAGATE_FAULT;
4196 rc = X86EMUL_CONTINUE;
4200 if (ops->get_msr(ctxt, c->regs[VCPU_REGS_RCX], &msr_data)) {
4201 emulate_gp(ctxt, 0);
4202 rc = X86EMUL_PROPAGATE_FAULT;
4205 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
4206 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
4208 rc = X86EMUL_CONTINUE;
4210 case 0x34: /* sysenter */
4211 rc = emulate_sysenter(ctxt, ops);
4213 case 0x35: /* sysexit */
4214 rc = emulate_sysexit(ctxt, ops);
4216 case 0x40 ... 0x4f: /* cmov */
4217 c->dst.val = c->dst.orig_val = c->src.val;
4218 if (!test_cc(c->b, ctxt->eflags))
4219 c->dst.type = OP_NONE; /* no writeback */
4221 case 0x80 ... 0x8f: /* jnz rel, etc*/
4222 if (test_cc(c->b, ctxt->eflags))
4223 jmp_rel(c, c->src.val);
4225 case 0x90 ... 0x9f: /* setcc r/m8 */
4226 c->dst.val = test_cc(c->b, ctxt->eflags);
4228 case 0xa0: /* push fs */
4229 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
4231 case 0xa1: /* pop fs */
4232 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
4236 c->dst.type = OP_NONE;
4237 /* only subword offset */
4238 c->src.val &= (c->dst.bytes << 3) - 1;
4239 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
4241 case 0xa4: /* shld imm8, r, r/m */
4242 case 0xa5: /* shld cl, r, r/m */
4243 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
4245 case 0xa8: /* push gs */
4246 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
4248 case 0xa9: /* pop gs */
4249 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
4253 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
4255 case 0xac: /* shrd imm8, r, r/m */
4256 case 0xad: /* shrd cl, r, r/m */
4257 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
4259 case 0xae: /* clflush */
4261 case 0xb0 ... 0xb1: /* cmpxchg */
4263 * Save real source value, then compare EAX against
4266 c->src.orig_val = c->src.val;
4267 c->src.val = c->regs[VCPU_REGS_RAX];
4268 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
4269 if (ctxt->eflags & EFLG_ZF) {
4270 /* Success: write back to memory. */
4271 c->dst.val = c->src.orig_val;
4273 /* Failure: write the value we saw to EAX. */
4274 c->dst.type = OP_REG;
4275 c->dst.addr.reg = (unsigned long *)&c->regs[VCPU_REGS_RAX];
4278 case 0xb2: /* lss */
4279 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_SS);
4283 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
4285 case 0xb4: /* lfs */
4286 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_FS);
4288 case 0xb5: /* lgs */
4289 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_GS);
4291 case 0xb6 ... 0xb7: /* movzx */
4292 c->dst.bytes = c->op_bytes;
4293 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
4296 case 0xba: /* Grp8 */
4297 switch (c->modrm_reg & 3) {
4310 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
4312 case 0xbc: { /* bsf */
4314 __asm__ ("bsf %2, %0; setz %1"
4315 : "=r"(c->dst.val), "=q"(zf)
4317 ctxt->eflags &= ~X86_EFLAGS_ZF;
4319 ctxt->eflags |= X86_EFLAGS_ZF;
4320 c->dst.type = OP_NONE; /* Disable writeback. */
4324 case 0xbd: { /* bsr */
4326 __asm__ ("bsr %2, %0; setz %1"
4327 : "=r"(c->dst.val), "=q"(zf)
4329 ctxt->eflags &= ~X86_EFLAGS_ZF;
4331 ctxt->eflags |= X86_EFLAGS_ZF;
4332 c->dst.type = OP_NONE; /* Disable writeback. */
4336 case 0xbe ... 0xbf: /* movsx */
4337 c->dst.bytes = c->op_bytes;
4338 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
4341 case 0xc0 ... 0xc1: /* xadd */
4342 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
4343 /* Write back the register source. */
4344 c->src.val = c->dst.orig_val;
4345 write_register_operand(&c->src);
4347 case 0xc3: /* movnti */
4348 c->dst.bytes = c->op_bytes;
4349 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
4352 case 0xc7: /* Grp9 (cmpxchg8b) */
4353 rc = emulate_grp9(ctxt, ops);
4356 goto cannot_emulate;
4359 if (rc != X86EMUL_CONTINUE)
4365 return EMULATION_FAILED;