1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
32 * Opcode effective-address decode tables.
33 * Note that we only emulate instructions that have at least one memory
34 * operand (excluding implicit stack references). We assume that stack
35 * references and instruction fetches will never occur in special memory
36 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
40 /* Operand sizes: 8-bit operands or specified/overridden size. */
41 #define ByteOp (1<<0) /* 8-bit operands. */
42 /* Destination operand type. */
43 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
44 #define DstReg (2<<1) /* Register operand. */
45 #define DstMem (3<<1) /* Memory operand. */
46 #define DstAcc (4<<1) /* Destination Accumulator */
47 #define DstDI (5<<1) /* Destination is in ES:(E)DI */
48 #define DstMem64 (6<<1) /* 64bit memory operand */
49 #define DstImmUByte (7<<1) /* 8-bit unsigned immediate operand */
50 #define DstMask (7<<1)
51 /* Source operand type. */
52 #define SrcNone (0<<4) /* No source operand. */
53 #define SrcReg (1<<4) /* Register operand. */
54 #define SrcMem (2<<4) /* Memory operand. */
55 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
56 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
57 #define SrcImm (5<<4) /* Immediate operand. */
58 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
59 #define SrcOne (7<<4) /* Implied '1' */
60 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
61 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
62 #define SrcSI (0xa<<4) /* Source is in the DS:RSI */
63 #define SrcImmFAddr (0xb<<4) /* Source is immediate far address */
64 #define SrcMemFAddr (0xc<<4) /* Source is far address in memory */
65 #define SrcAcc (0xd<<4) /* Source Accumulator */
66 #define SrcImmU16 (0xe<<4) /* Immediate operand, unsigned, 16 bits */
67 #define SrcMask (0xf<<4)
68 /* Generic ModRM decode. */
70 /* Destination is only written; never read. */
73 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
74 #define String (1<<12) /* String instruction (rep capable) */
75 #define Stack (1<<13) /* Stack instruction (push/pop) */
76 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
77 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
78 #define Prefix (1<<16) /* Instruction varies with 66/f2/f3 prefix */
79 #define Sse (1<<17) /* SSE Vector instruction */
80 #define RMExt (1<<18) /* Opcode extension in ModRM r/m if mod == 3 */
82 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
83 #define VendorSpecific (1<<22) /* Vendor specific instruction */
84 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
85 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
86 #define Undefined (1<<25) /* No Such Instruction */
87 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
88 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
90 /* Source 2 operand type */
91 #define Src2None (0<<29)
92 #define Src2CL (1<<29)
93 #define Src2ImmByte (2<<29)
94 #define Src2One (3<<29)
95 #define Src2Imm (4<<29)
96 #define Src2Mask (7<<29)
99 #define X3(x...) X2(x), x
100 #define X4(x...) X2(x), X2(x)
101 #define X5(x...) X4(x), x
102 #define X6(x...) X4(x), X2(x)
103 #define X7(x...) X4(x), X3(x)
104 #define X8(x...) X4(x), X4(x)
105 #define X16(x...) X8(x), X8(x)
111 int (*execute)(struct x86_emulate_ctxt *ctxt);
112 struct opcode *group;
113 struct group_dual *gdual;
114 struct gprefix *gprefix;
116 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
120 struct opcode mod012[8];
121 struct opcode mod3[8];
125 struct opcode pfx_no;
126 struct opcode pfx_66;
127 struct opcode pfx_f2;
128 struct opcode pfx_f3;
131 /* EFLAGS bit definitions. */
132 #define EFLG_ID (1<<21)
133 #define EFLG_VIP (1<<20)
134 #define EFLG_VIF (1<<19)
135 #define EFLG_AC (1<<18)
136 #define EFLG_VM (1<<17)
137 #define EFLG_RF (1<<16)
138 #define EFLG_IOPL (3<<12)
139 #define EFLG_NT (1<<14)
140 #define EFLG_OF (1<<11)
141 #define EFLG_DF (1<<10)
142 #define EFLG_IF (1<<9)
143 #define EFLG_TF (1<<8)
144 #define EFLG_SF (1<<7)
145 #define EFLG_ZF (1<<6)
146 #define EFLG_AF (1<<4)
147 #define EFLG_PF (1<<2)
148 #define EFLG_CF (1<<0)
150 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
151 #define EFLG_RESERVED_ONE_MASK 2
154 * Instruction emulation:
155 * Most instructions are emulated directly via a fragment of inline assembly
156 * code. This allows us to save/restore EFLAGS and thus very easily pick up
157 * any modified flags.
160 #if defined(CONFIG_X86_64)
161 #define _LO32 "k" /* force 32-bit operand */
162 #define _STK "%%rsp" /* stack pointer */
163 #elif defined(__i386__)
164 #define _LO32 "" /* force 32-bit operand */
165 #define _STK "%%esp" /* stack pointer */
169 * These EFLAGS bits are restored from saved value during emulation, and
170 * any changes are written back to the saved value after emulation.
172 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
174 /* Before executing instruction: restore necessary bits in EFLAGS. */
175 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
176 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
177 "movl %"_sav",%"_LO32 _tmp"; " \
180 "movl %"_msk",%"_LO32 _tmp"; " \
181 "andl %"_LO32 _tmp",("_STK"); " \
183 "notl %"_LO32 _tmp"; " \
184 "andl %"_LO32 _tmp",("_STK"); " \
185 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
187 "orl %"_LO32 _tmp",("_STK"); " \
191 /* After executing instruction: write-back necessary bits in EFLAGS. */
192 #define _POST_EFLAGS(_sav, _msk, _tmp) \
193 /* _sav |= EFLAGS & _msk; */ \
196 "andl %"_msk",%"_LO32 _tmp"; " \
197 "orl %"_LO32 _tmp",%"_sav"; "
205 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix, _dsttype) \
207 __asm__ __volatile__ ( \
208 _PRE_EFLAGS("0", "4", "2") \
209 _op _suffix " %"_x"3,%1; " \
210 _POST_EFLAGS("0", "4", "2") \
211 : "=m" (_eflags), "+q" (*(_dsttype*)&(_dst).val),\
213 : _y ((_src).val), "i" (EFLAGS_MASK)); \
217 /* Raw emulation: instruction has two explicit operands. */
218 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
220 unsigned long _tmp; \
222 switch ((_dst).bytes) { \
224 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w",u16);\
227 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l",u32);\
230 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q",u64)); \
235 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
237 unsigned long _tmp; \
238 switch ((_dst).bytes) { \
240 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b",u8); \
243 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
244 _wx, _wy, _lx, _ly, _qx, _qy); \
249 /* Source operand is byte-sized and may be restricted to just %cl. */
250 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
251 __emulate_2op(_op, _src, _dst, _eflags, \
252 "b", "c", "b", "c", "b", "c", "b", "c")
254 /* Source operand is byte, word, long or quad sized. */
255 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
256 __emulate_2op(_op, _src, _dst, _eflags, \
257 "b", "q", "w", "r", _LO32, "r", "", "r")
259 /* Source operand is word, long or quad sized. */
260 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
261 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
262 "w", "r", _LO32, "r", "", "r")
264 /* Instruction has three operands and one operand is stored in ECX register */
265 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
267 unsigned long _tmp; \
268 _type _clv = (_cl).val; \
269 _type _srcv = (_src).val; \
270 _type _dstv = (_dst).val; \
272 __asm__ __volatile__ ( \
273 _PRE_EFLAGS("0", "5", "2") \
274 _op _suffix " %4,%1 \n" \
275 _POST_EFLAGS("0", "5", "2") \
276 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
277 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
280 (_cl).val = (unsigned long) _clv; \
281 (_src).val = (unsigned long) _srcv; \
282 (_dst).val = (unsigned long) _dstv; \
285 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
287 switch ((_dst).bytes) { \
289 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
290 "w", unsigned short); \
293 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
294 "l", unsigned int); \
297 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
298 "q", unsigned long)); \
303 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
305 unsigned long _tmp; \
307 __asm__ __volatile__ ( \
308 _PRE_EFLAGS("0", "3", "2") \
309 _op _suffix " %1; " \
310 _POST_EFLAGS("0", "3", "2") \
311 : "=m" (_eflags), "+m" ((_dst).val), \
313 : "i" (EFLAGS_MASK)); \
316 /* Instruction has only one explicit operand (no source operand). */
317 #define emulate_1op(_op, _dst, _eflags) \
319 switch ((_dst).bytes) { \
320 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
321 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
322 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
323 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
327 #define __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, _suffix) \
329 unsigned long _tmp; \
331 __asm__ __volatile__ ( \
332 _PRE_EFLAGS("0", "4", "1") \
333 _op _suffix " %5; " \
334 _POST_EFLAGS("0", "4", "1") \
335 : "=m" (_eflags), "=&r" (_tmp), \
336 "+a" (_rax), "+d" (_rdx) \
337 : "i" (EFLAGS_MASK), "m" ((_src).val), \
338 "a" (_rax), "d" (_rdx)); \
341 #define __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _suffix, _ex) \
343 unsigned long _tmp; \
345 __asm__ __volatile__ ( \
346 _PRE_EFLAGS("0", "5", "1") \
348 _op _suffix " %6; " \
350 _POST_EFLAGS("0", "5", "1") \
351 ".pushsection .fixup,\"ax\" \n\t" \
352 "3: movb $1, %4 \n\t" \
355 _ASM_EXTABLE(1b, 3b) \
356 : "=m" (_eflags), "=&r" (_tmp), \
357 "+a" (_rax), "+d" (_rdx), "+qm"(_ex) \
358 : "i" (EFLAGS_MASK), "m" ((_src).val), \
359 "a" (_rax), "d" (_rdx)); \
362 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
363 #define emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags) \
365 switch((_src).bytes) { \
367 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
371 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
375 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
379 ON64(__emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
385 #define emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _ex) \
387 switch((_src).bytes) { \
389 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
390 _eflags, "b", _ex); \
393 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
394 _eflags, "w", _ex); \
397 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
398 _eflags, "l", _ex); \
401 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
402 _eflags, "q", _ex)); \
407 /* Fetch next part of the instruction being emulated. */
408 #define insn_fetch(_type, _size, _eip) \
409 ({ unsigned long _x; \
410 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
411 if (rc != X86EMUL_CONTINUE) \
417 #define insn_fetch_arr(_arr, _size, _eip) \
418 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
419 if (rc != X86EMUL_CONTINUE) \
424 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
425 enum x86_intercept intercept,
426 enum x86_intercept_stage stage)
428 struct x86_instruction_info info = {
429 .intercept = intercept,
430 .rep_prefix = ctxt->decode.rep_prefix,
431 .modrm_mod = ctxt->decode.modrm_mod,
432 .modrm_reg = ctxt->decode.modrm_reg,
433 .modrm_rm = ctxt->decode.modrm_rm,
434 .src_val = ctxt->decode.src.val64,
435 .src_bytes = ctxt->decode.src.bytes,
436 .dst_bytes = ctxt->decode.dst.bytes,
437 .ad_bytes = ctxt->decode.ad_bytes,
438 .next_rip = ctxt->eip,
441 return ctxt->ops->intercept(ctxt->vcpu, &info, stage);
444 static inline unsigned long ad_mask(struct decode_cache *c)
446 return (1UL << (c->ad_bytes << 3)) - 1;
449 /* Access/update address held in a register, based on addressing mode. */
450 static inline unsigned long
451 address_mask(struct decode_cache *c, unsigned long reg)
453 if (c->ad_bytes == sizeof(unsigned long))
456 return reg & ad_mask(c);
459 static inline unsigned long
460 register_address(struct decode_cache *c, unsigned long reg)
462 return address_mask(c, reg);
466 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
468 if (c->ad_bytes == sizeof(unsigned long))
471 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
474 static inline void jmp_rel(struct decode_cache *c, int rel)
476 register_address_increment(c, &c->eip, rel);
479 static u32 desc_limit_scaled(struct desc_struct *desc)
481 u32 limit = get_desc_limit(desc);
483 return desc->g ? (limit << 12) | 0xfff : limit;
486 static void set_seg_override(struct decode_cache *c, int seg)
488 c->has_seg_override = true;
489 c->seg_override = seg;
492 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
493 struct x86_emulate_ops *ops, int seg)
495 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
498 return ops->get_cached_segment_base(seg, ctxt->vcpu);
501 static unsigned seg_override(struct x86_emulate_ctxt *ctxt,
502 struct x86_emulate_ops *ops,
503 struct decode_cache *c)
505 if (!c->has_seg_override)
508 return c->seg_override;
511 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
512 u32 error, bool valid)
514 ctxt->exception.vector = vec;
515 ctxt->exception.error_code = error;
516 ctxt->exception.error_code_valid = valid;
517 return X86EMUL_PROPAGATE_FAULT;
520 static int emulate_db(struct x86_emulate_ctxt *ctxt)
522 return emulate_exception(ctxt, DB_VECTOR, 0, false);
525 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
527 return emulate_exception(ctxt, GP_VECTOR, err, true);
530 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
532 return emulate_exception(ctxt, SS_VECTOR, err, true);
535 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
537 return emulate_exception(ctxt, UD_VECTOR, 0, false);
540 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
542 return emulate_exception(ctxt, TS_VECTOR, err, true);
545 static int emulate_de(struct x86_emulate_ctxt *ctxt)
547 return emulate_exception(ctxt, DE_VECTOR, 0, false);
550 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
552 return emulate_exception(ctxt, NM_VECTOR, 0, false);
555 static int __linearize(struct x86_emulate_ctxt *ctxt,
556 struct segmented_address addr,
557 unsigned size, bool write, bool fetch,
560 struct decode_cache *c = &ctxt->decode;
561 struct desc_struct desc;
567 la = seg_base(ctxt, ctxt->ops, addr.seg) + addr.ea;
568 switch (ctxt->mode) {
569 case X86EMUL_MODE_REAL:
571 case X86EMUL_MODE_PROT64:
572 if (((signed long)la << 16) >> 16 != la)
573 return emulate_gp(ctxt, 0);
576 usable = ctxt->ops->get_cached_descriptor(&desc, NULL, addr.seg,
580 /* code segment or read-only data segment */
581 if (((desc.type & 8) || !(desc.type & 2)) && write)
583 /* unreadable code segment */
584 if (!fetch && (desc.type & 8) && !(desc.type & 2))
586 lim = desc_limit_scaled(&desc);
587 if ((desc.type & 8) || !(desc.type & 4)) {
588 /* expand-up segment */
589 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
592 /* exapand-down segment */
593 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
595 lim = desc.d ? 0xffffffff : 0xffff;
596 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
599 cpl = ctxt->ops->cpl(ctxt->vcpu);
600 rpl = ctxt->ops->get_segment_selector(addr.seg, ctxt->vcpu) & 3;
602 if (!(desc.type & 8)) {
606 } else if ((desc.type & 8) && !(desc.type & 4)) {
607 /* nonconforming code segment */
610 } else if ((desc.type & 8) && (desc.type & 4)) {
611 /* conforming code segment */
617 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : c->ad_bytes != 8)
620 return X86EMUL_CONTINUE;
622 if (addr.seg == VCPU_SREG_SS)
623 return emulate_ss(ctxt, addr.seg);
625 return emulate_gp(ctxt, addr.seg);
628 static int linearize(struct x86_emulate_ctxt *ctxt,
629 struct segmented_address addr,
630 unsigned size, bool write,
633 return __linearize(ctxt, addr, size, write, false, linear);
637 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
638 struct segmented_address addr,
645 rc = linearize(ctxt, addr, size, false, &linear);
646 if (rc != X86EMUL_CONTINUE)
648 return ctxt->ops->read_std(linear, data, size, ctxt->vcpu,
652 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
653 struct x86_emulate_ops *ops,
654 unsigned long eip, u8 *dest)
656 struct fetch_cache *fc = &ctxt->decode.fetch;
660 if (eip == fc->end) {
661 unsigned long linear;
662 struct segmented_address addr = { .seg=VCPU_SREG_CS, .ea=eip};
663 cur_size = fc->end - fc->start;
664 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
665 rc = __linearize(ctxt, addr, size, false, true, &linear);
666 if (rc != X86EMUL_CONTINUE)
668 rc = ops->fetch(linear, fc->data + cur_size,
669 size, ctxt->vcpu, &ctxt->exception);
670 if (rc != X86EMUL_CONTINUE)
674 *dest = fc->data[eip - fc->start];
675 return X86EMUL_CONTINUE;
678 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
679 struct x86_emulate_ops *ops,
680 unsigned long eip, void *dest, unsigned size)
684 /* x86 instructions are limited to 15 bytes. */
685 if (eip + size - ctxt->eip > 15)
686 return X86EMUL_UNHANDLEABLE;
688 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
689 if (rc != X86EMUL_CONTINUE)
692 return X86EMUL_CONTINUE;
696 * Given the 'reg' portion of a ModRM byte, and a register block, return a
697 * pointer into the block that addresses the relevant register.
698 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
700 static void *decode_register(u8 modrm_reg, unsigned long *regs,
705 p = ®s[modrm_reg];
706 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
707 p = (unsigned char *)®s[modrm_reg & 3] + 1;
711 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
712 struct x86_emulate_ops *ops,
713 struct segmented_address addr,
714 u16 *size, unsigned long *address, int op_bytes)
721 rc = segmented_read_std(ctxt, addr, size, 2);
722 if (rc != X86EMUL_CONTINUE)
725 rc = segmented_read_std(ctxt, addr, address, op_bytes);
729 static int test_cc(unsigned int condition, unsigned int flags)
733 switch ((condition & 15) >> 1) {
735 rc |= (flags & EFLG_OF);
737 case 1: /* b/c/nae */
738 rc |= (flags & EFLG_CF);
741 rc |= (flags & EFLG_ZF);
744 rc |= (flags & (EFLG_CF|EFLG_ZF));
747 rc |= (flags & EFLG_SF);
750 rc |= (flags & EFLG_PF);
753 rc |= (flags & EFLG_ZF);
756 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
760 /* Odd condition identifiers (lsb == 1) have inverted sense. */
761 return (!!rc ^ (condition & 1));
764 static void fetch_register_operand(struct operand *op)
768 op->val = *(u8 *)op->addr.reg;
771 op->val = *(u16 *)op->addr.reg;
774 op->val = *(u32 *)op->addr.reg;
777 op->val = *(u64 *)op->addr.reg;
782 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
784 ctxt->ops->get_fpu(ctxt);
786 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
787 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
788 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
789 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
790 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
791 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
792 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
793 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
795 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
796 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
797 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
798 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
799 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
800 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
801 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
802 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
806 ctxt->ops->put_fpu(ctxt);
809 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
812 ctxt->ops->get_fpu(ctxt);
814 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
815 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
816 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
817 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
818 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
819 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
820 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
821 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
823 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
824 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
825 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
826 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
827 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
828 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
829 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
830 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
834 ctxt->ops->put_fpu(ctxt);
837 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
839 struct decode_cache *c,
842 unsigned reg = c->modrm_reg;
843 int highbyte_regs = c->rex_prefix == 0;
846 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
852 read_sse_reg(ctxt, &op->vec_val, reg);
857 if ((c->d & ByteOp) && !inhibit_bytereg) {
858 op->addr.reg = decode_register(reg, c->regs, highbyte_regs);
861 op->addr.reg = decode_register(reg, c->regs, 0);
862 op->bytes = c->op_bytes;
864 fetch_register_operand(op);
865 op->orig_val = op->val;
868 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
869 struct x86_emulate_ops *ops,
872 struct decode_cache *c = &ctxt->decode;
874 int index_reg = 0, base_reg = 0, scale;
875 int rc = X86EMUL_CONTINUE;
879 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
880 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
881 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
884 c->modrm = insn_fetch(u8, 1, c->eip);
885 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
886 c->modrm_reg |= (c->modrm & 0x38) >> 3;
887 c->modrm_rm |= (c->modrm & 0x07);
888 c->modrm_seg = VCPU_SREG_DS;
890 if (c->modrm_mod == 3) {
892 op->bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
893 op->addr.reg = decode_register(c->modrm_rm,
894 c->regs, c->d & ByteOp);
898 op->addr.xmm = c->modrm_rm;
899 read_sse_reg(ctxt, &op->vec_val, c->modrm_rm);
902 fetch_register_operand(op);
908 if (c->ad_bytes == 2) {
909 unsigned bx = c->regs[VCPU_REGS_RBX];
910 unsigned bp = c->regs[VCPU_REGS_RBP];
911 unsigned si = c->regs[VCPU_REGS_RSI];
912 unsigned di = c->regs[VCPU_REGS_RDI];
914 /* 16-bit ModR/M decode. */
915 switch (c->modrm_mod) {
917 if (c->modrm_rm == 6)
918 modrm_ea += insn_fetch(u16, 2, c->eip);
921 modrm_ea += insn_fetch(s8, 1, c->eip);
924 modrm_ea += insn_fetch(u16, 2, c->eip);
927 switch (c->modrm_rm) {
947 if (c->modrm_mod != 0)
954 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
955 (c->modrm_rm == 6 && c->modrm_mod != 0))
956 c->modrm_seg = VCPU_SREG_SS;
957 modrm_ea = (u16)modrm_ea;
959 /* 32/64-bit ModR/M decode. */
960 if ((c->modrm_rm & 7) == 4) {
961 sib = insn_fetch(u8, 1, c->eip);
962 index_reg |= (sib >> 3) & 7;
966 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
967 modrm_ea += insn_fetch(s32, 4, c->eip);
969 modrm_ea += c->regs[base_reg];
971 modrm_ea += c->regs[index_reg] << scale;
972 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
973 if (ctxt->mode == X86EMUL_MODE_PROT64)
976 modrm_ea += c->regs[c->modrm_rm];
977 switch (c->modrm_mod) {
979 if (c->modrm_rm == 5)
980 modrm_ea += insn_fetch(s32, 4, c->eip);
983 modrm_ea += insn_fetch(s8, 1, c->eip);
986 modrm_ea += insn_fetch(s32, 4, c->eip);
990 op->addr.mem.ea = modrm_ea;
995 static int decode_abs(struct x86_emulate_ctxt *ctxt,
996 struct x86_emulate_ops *ops,
999 struct decode_cache *c = &ctxt->decode;
1000 int rc = X86EMUL_CONTINUE;
1003 switch (c->ad_bytes) {
1005 op->addr.mem.ea = insn_fetch(u16, 2, c->eip);
1008 op->addr.mem.ea = insn_fetch(u32, 4, c->eip);
1011 op->addr.mem.ea = insn_fetch(u64, 8, c->eip);
1018 static void fetch_bit_operand(struct decode_cache *c)
1022 if (c->dst.type == OP_MEM && c->src.type == OP_REG) {
1023 mask = ~(c->dst.bytes * 8 - 1);
1025 if (c->src.bytes == 2)
1026 sv = (s16)c->src.val & (s16)mask;
1027 else if (c->src.bytes == 4)
1028 sv = (s32)c->src.val & (s32)mask;
1030 c->dst.addr.mem.ea += (sv >> 3);
1033 /* only subword offset */
1034 c->src.val &= (c->dst.bytes << 3) - 1;
1037 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1038 struct x86_emulate_ops *ops,
1039 unsigned long addr, void *dest, unsigned size)
1042 struct read_cache *mc = &ctxt->decode.mem_read;
1045 int n = min(size, 8u);
1047 if (mc->pos < mc->end)
1050 rc = ops->read_emulated(addr, mc->data + mc->end, n,
1051 &ctxt->exception, ctxt->vcpu);
1052 if (rc != X86EMUL_CONTINUE)
1057 memcpy(dest, mc->data + mc->pos, n);
1062 return X86EMUL_CONTINUE;
1065 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1066 struct segmented_address addr,
1073 rc = linearize(ctxt, addr, size, false, &linear);
1074 if (rc != X86EMUL_CONTINUE)
1076 return read_emulated(ctxt, ctxt->ops, linear, data, size);
1079 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1080 struct segmented_address addr,
1087 rc = linearize(ctxt, addr, size, true, &linear);
1088 if (rc != X86EMUL_CONTINUE)
1090 return ctxt->ops->write_emulated(linear, data, size,
1091 &ctxt->exception, ctxt->vcpu);
1094 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1095 struct segmented_address addr,
1096 const void *orig_data, const void *data,
1102 rc = linearize(ctxt, addr, size, true, &linear);
1103 if (rc != X86EMUL_CONTINUE)
1105 return ctxt->ops->cmpxchg_emulated(linear, orig_data, data,
1106 size, &ctxt->exception, ctxt->vcpu);
1109 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1110 struct x86_emulate_ops *ops,
1111 unsigned int size, unsigned short port,
1114 struct read_cache *rc = &ctxt->decode.io_read;
1116 if (rc->pos == rc->end) { /* refill pio read ahead */
1117 struct decode_cache *c = &ctxt->decode;
1118 unsigned int in_page, n;
1119 unsigned int count = c->rep_prefix ?
1120 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
1121 in_page = (ctxt->eflags & EFLG_DF) ?
1122 offset_in_page(c->regs[VCPU_REGS_RDI]) :
1123 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
1124 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1128 rc->pos = rc->end = 0;
1129 if (!ops->pio_in_emulated(size, port, rc->data, n, ctxt->vcpu))
1134 memcpy(dest, rc->data + rc->pos, size);
1139 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1140 struct x86_emulate_ops *ops,
1141 u16 selector, struct desc_ptr *dt)
1143 if (selector & 1 << 2) {
1144 struct desc_struct desc;
1145 memset (dt, 0, sizeof *dt);
1146 if (!ops->get_cached_descriptor(&desc, NULL, VCPU_SREG_LDTR,
1150 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1151 dt->address = get_desc_base(&desc);
1153 ops->get_gdt(dt, ctxt->vcpu);
1156 /* allowed just for 8 bytes segments */
1157 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1158 struct x86_emulate_ops *ops,
1159 u16 selector, struct desc_struct *desc)
1162 u16 index = selector >> 3;
1166 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1168 if (dt.size < index * 8 + 7)
1169 return emulate_gp(ctxt, selector & 0xfffc);
1170 addr = dt.address + index * 8;
1171 ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu,
1177 /* allowed just for 8 bytes segments */
1178 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1179 struct x86_emulate_ops *ops,
1180 u16 selector, struct desc_struct *desc)
1183 u16 index = selector >> 3;
1187 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1189 if (dt.size < index * 8 + 7)
1190 return emulate_gp(ctxt, selector & 0xfffc);
1192 addr = dt.address + index * 8;
1193 ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu,
1199 /* Does not support long mode */
1200 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1201 struct x86_emulate_ops *ops,
1202 u16 selector, int seg)
1204 struct desc_struct seg_desc;
1206 unsigned err_vec = GP_VECTOR;
1208 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1211 memset(&seg_desc, 0, sizeof seg_desc);
1213 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1214 || ctxt->mode == X86EMUL_MODE_REAL) {
1215 /* set real mode segment descriptor */
1216 set_desc_base(&seg_desc, selector << 4);
1217 set_desc_limit(&seg_desc, 0xffff);
1224 /* NULL selector is not valid for TR, CS and SS */
1225 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1229 /* TR should be in GDT only */
1230 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1233 if (null_selector) /* for NULL selector skip all following checks */
1236 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1237 if (ret != X86EMUL_CONTINUE)
1240 err_code = selector & 0xfffc;
1241 err_vec = GP_VECTOR;
1243 /* can't load system descriptor into segment selecor */
1244 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1248 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1254 cpl = ops->cpl(ctxt->vcpu);
1259 * segment is not a writable data segment or segment
1260 * selector's RPL != CPL or segment selector's RPL != CPL
1262 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1266 if (!(seg_desc.type & 8))
1269 if (seg_desc.type & 4) {
1275 if (rpl > cpl || dpl != cpl)
1278 /* CS(RPL) <- CPL */
1279 selector = (selector & 0xfffc) | cpl;
1282 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1285 case VCPU_SREG_LDTR:
1286 if (seg_desc.s || seg_desc.type != 2)
1289 default: /* DS, ES, FS, or GS */
1291 * segment is not a data or readable code segment or
1292 * ((segment is a data or nonconforming code segment)
1293 * and (both RPL and CPL > DPL))
1295 if ((seg_desc.type & 0xa) == 0x8 ||
1296 (((seg_desc.type & 0xc) != 0xc) &&
1297 (rpl > dpl && cpl > dpl)))
1303 /* mark segment as accessed */
1305 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1306 if (ret != X86EMUL_CONTINUE)
1310 ops->set_segment_selector(selector, seg, ctxt->vcpu);
1311 ops->set_cached_descriptor(&seg_desc, 0, seg, ctxt->vcpu);
1312 return X86EMUL_CONTINUE;
1314 emulate_exception(ctxt, err_vec, err_code, true);
1315 return X86EMUL_PROPAGATE_FAULT;
1318 static void write_register_operand(struct operand *op)
1320 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1321 switch (op->bytes) {
1323 *(u8 *)op->addr.reg = (u8)op->val;
1326 *(u16 *)op->addr.reg = (u16)op->val;
1329 *op->addr.reg = (u32)op->val;
1330 break; /* 64b: zero-extend */
1332 *op->addr.reg = op->val;
1337 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1338 struct x86_emulate_ops *ops)
1341 struct decode_cache *c = &ctxt->decode;
1343 switch (c->dst.type) {
1345 write_register_operand(&c->dst);
1349 rc = segmented_cmpxchg(ctxt,
1355 rc = segmented_write(ctxt,
1359 if (rc != X86EMUL_CONTINUE)
1363 write_sse_reg(ctxt, &c->dst.vec_val, c->dst.addr.xmm);
1371 return X86EMUL_CONTINUE;
1374 static int em_push(struct x86_emulate_ctxt *ctxt)
1376 struct decode_cache *c = &ctxt->decode;
1377 struct segmented_address addr;
1379 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1380 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1381 addr.seg = VCPU_SREG_SS;
1383 /* Disable writeback. */
1384 c->dst.type = OP_NONE;
1385 return segmented_write(ctxt, addr, &c->src.val, c->op_bytes);
1388 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1389 struct x86_emulate_ops *ops,
1390 void *dest, int len)
1392 struct decode_cache *c = &ctxt->decode;
1394 struct segmented_address addr;
1396 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1397 addr.seg = VCPU_SREG_SS;
1398 rc = segmented_read(ctxt, addr, dest, len);
1399 if (rc != X86EMUL_CONTINUE)
1402 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1406 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1407 struct x86_emulate_ops *ops,
1408 void *dest, int len)
1411 unsigned long val, change_mask;
1412 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1413 int cpl = ops->cpl(ctxt->vcpu);
1415 rc = emulate_pop(ctxt, ops, &val, len);
1416 if (rc != X86EMUL_CONTINUE)
1419 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1420 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1422 switch(ctxt->mode) {
1423 case X86EMUL_MODE_PROT64:
1424 case X86EMUL_MODE_PROT32:
1425 case X86EMUL_MODE_PROT16:
1427 change_mask |= EFLG_IOPL;
1429 change_mask |= EFLG_IF;
1431 case X86EMUL_MODE_VM86:
1433 return emulate_gp(ctxt, 0);
1434 change_mask |= EFLG_IF;
1436 default: /* real mode */
1437 change_mask |= (EFLG_IOPL | EFLG_IF);
1441 *(unsigned long *)dest =
1442 (ctxt->eflags & ~change_mask) | (val & change_mask);
1447 static int emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1448 struct x86_emulate_ops *ops, int seg)
1450 struct decode_cache *c = &ctxt->decode;
1452 c->src.val = ops->get_segment_selector(seg, ctxt->vcpu);
1454 return em_push(ctxt);
1457 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1458 struct x86_emulate_ops *ops, int seg)
1460 struct decode_cache *c = &ctxt->decode;
1461 unsigned long selector;
1464 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1465 if (rc != X86EMUL_CONTINUE)
1468 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1472 static int emulate_pusha(struct x86_emulate_ctxt *ctxt)
1474 struct decode_cache *c = &ctxt->decode;
1475 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1476 int rc = X86EMUL_CONTINUE;
1477 int reg = VCPU_REGS_RAX;
1479 while (reg <= VCPU_REGS_RDI) {
1480 (reg == VCPU_REGS_RSP) ?
1481 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1484 if (rc != X86EMUL_CONTINUE)
1493 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1494 struct x86_emulate_ops *ops)
1496 struct decode_cache *c = &ctxt->decode;
1497 int rc = X86EMUL_CONTINUE;
1498 int reg = VCPU_REGS_RDI;
1500 while (reg >= VCPU_REGS_RAX) {
1501 if (reg == VCPU_REGS_RSP) {
1502 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1507 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1508 if (rc != X86EMUL_CONTINUE)
1515 int emulate_int_real(struct x86_emulate_ctxt *ctxt,
1516 struct x86_emulate_ops *ops, int irq)
1518 struct decode_cache *c = &ctxt->decode;
1525 /* TODO: Add limit checks */
1526 c->src.val = ctxt->eflags;
1528 if (rc != X86EMUL_CONTINUE)
1531 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1533 c->src.val = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
1535 if (rc != X86EMUL_CONTINUE)
1538 c->src.val = c->eip;
1540 if (rc != X86EMUL_CONTINUE)
1543 ops->get_idt(&dt, ctxt->vcpu);
1545 eip_addr = dt.address + (irq << 2);
1546 cs_addr = dt.address + (irq << 2) + 2;
1548 rc = ops->read_std(cs_addr, &cs, 2, ctxt->vcpu, &ctxt->exception);
1549 if (rc != X86EMUL_CONTINUE)
1552 rc = ops->read_std(eip_addr, &eip, 2, ctxt->vcpu, &ctxt->exception);
1553 if (rc != X86EMUL_CONTINUE)
1556 rc = load_segment_descriptor(ctxt, ops, cs, VCPU_SREG_CS);
1557 if (rc != X86EMUL_CONTINUE)
1565 static int emulate_int(struct x86_emulate_ctxt *ctxt,
1566 struct x86_emulate_ops *ops, int irq)
1568 switch(ctxt->mode) {
1569 case X86EMUL_MODE_REAL:
1570 return emulate_int_real(ctxt, ops, irq);
1571 case X86EMUL_MODE_VM86:
1572 case X86EMUL_MODE_PROT16:
1573 case X86EMUL_MODE_PROT32:
1574 case X86EMUL_MODE_PROT64:
1576 /* Protected mode interrupts unimplemented yet */
1577 return X86EMUL_UNHANDLEABLE;
1581 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
1582 struct x86_emulate_ops *ops)
1584 struct decode_cache *c = &ctxt->decode;
1585 int rc = X86EMUL_CONTINUE;
1586 unsigned long temp_eip = 0;
1587 unsigned long temp_eflags = 0;
1588 unsigned long cs = 0;
1589 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1590 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1591 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1592 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1594 /* TODO: Add stack limit check */
1596 rc = emulate_pop(ctxt, ops, &temp_eip, c->op_bytes);
1598 if (rc != X86EMUL_CONTINUE)
1601 if (temp_eip & ~0xffff)
1602 return emulate_gp(ctxt, 0);
1604 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1606 if (rc != X86EMUL_CONTINUE)
1609 rc = emulate_pop(ctxt, ops, &temp_eflags, c->op_bytes);
1611 if (rc != X86EMUL_CONTINUE)
1614 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1616 if (rc != X86EMUL_CONTINUE)
1622 if (c->op_bytes == 4)
1623 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1624 else if (c->op_bytes == 2) {
1625 ctxt->eflags &= ~0xffff;
1626 ctxt->eflags |= temp_eflags;
1629 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1630 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1635 static inline int emulate_iret(struct x86_emulate_ctxt *ctxt,
1636 struct x86_emulate_ops* ops)
1638 switch(ctxt->mode) {
1639 case X86EMUL_MODE_REAL:
1640 return emulate_iret_real(ctxt, ops);
1641 case X86EMUL_MODE_VM86:
1642 case X86EMUL_MODE_PROT16:
1643 case X86EMUL_MODE_PROT32:
1644 case X86EMUL_MODE_PROT64:
1646 /* iret from protected mode unimplemented yet */
1647 return X86EMUL_UNHANDLEABLE;
1651 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1652 struct x86_emulate_ops *ops)
1654 struct decode_cache *c = &ctxt->decode;
1656 return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1659 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1661 struct decode_cache *c = &ctxt->decode;
1662 switch (c->modrm_reg) {
1664 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1667 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1670 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1673 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1675 case 4: /* sal/shl */
1676 case 6: /* sal/shl */
1677 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1680 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1683 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1688 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1689 struct x86_emulate_ops *ops)
1691 struct decode_cache *c = &ctxt->decode;
1692 unsigned long *rax = &c->regs[VCPU_REGS_RAX];
1693 unsigned long *rdx = &c->regs[VCPU_REGS_RDX];
1696 switch (c->modrm_reg) {
1697 case 0 ... 1: /* test */
1698 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1701 c->dst.val = ~c->dst.val;
1704 emulate_1op("neg", c->dst, ctxt->eflags);
1707 emulate_1op_rax_rdx("mul", c->src, *rax, *rdx, ctxt->eflags);
1710 emulate_1op_rax_rdx("imul", c->src, *rax, *rdx, ctxt->eflags);
1713 emulate_1op_rax_rdx_ex("div", c->src, *rax, *rdx,
1717 emulate_1op_rax_rdx_ex("idiv", c->src, *rax, *rdx,
1721 return X86EMUL_UNHANDLEABLE;
1724 return emulate_de(ctxt);
1725 return X86EMUL_CONTINUE;
1728 static int emulate_grp45(struct x86_emulate_ctxt *ctxt)
1730 struct decode_cache *c = &ctxt->decode;
1731 int rc = X86EMUL_CONTINUE;
1733 switch (c->modrm_reg) {
1735 emulate_1op("inc", c->dst, ctxt->eflags);
1738 emulate_1op("dec", c->dst, ctxt->eflags);
1740 case 2: /* call near abs */ {
1743 c->eip = c->src.val;
1744 c->src.val = old_eip;
1748 case 4: /* jmp abs */
1749 c->eip = c->src.val;
1758 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1759 struct x86_emulate_ops *ops)
1761 struct decode_cache *c = &ctxt->decode;
1762 u64 old = c->dst.orig_val64;
1764 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1765 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1766 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1767 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1768 ctxt->eflags &= ~EFLG_ZF;
1770 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1771 (u32) c->regs[VCPU_REGS_RBX];
1773 ctxt->eflags |= EFLG_ZF;
1775 return X86EMUL_CONTINUE;
1778 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1779 struct x86_emulate_ops *ops)
1781 struct decode_cache *c = &ctxt->decode;
1785 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1786 if (rc != X86EMUL_CONTINUE)
1788 if (c->op_bytes == 4)
1789 c->eip = (u32)c->eip;
1790 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1791 if (rc != X86EMUL_CONTINUE)
1793 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1797 static int emulate_load_segment(struct x86_emulate_ctxt *ctxt,
1798 struct x86_emulate_ops *ops, int seg)
1800 struct decode_cache *c = &ctxt->decode;
1804 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
1806 rc = load_segment_descriptor(ctxt, ops, sel, seg);
1807 if (rc != X86EMUL_CONTINUE)
1810 c->dst.val = c->src.val;
1815 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1816 struct x86_emulate_ops *ops, struct desc_struct *cs,
1817 struct desc_struct *ss)
1819 memset(cs, 0, sizeof(struct desc_struct));
1820 ops->get_cached_descriptor(cs, NULL, VCPU_SREG_CS, ctxt->vcpu);
1821 memset(ss, 0, sizeof(struct desc_struct));
1823 cs->l = 0; /* will be adjusted later */
1824 set_desc_base(cs, 0); /* flat segment */
1825 cs->g = 1; /* 4kb granularity */
1826 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1827 cs->type = 0x0b; /* Read, Execute, Accessed */
1829 cs->dpl = 0; /* will be adjusted later */
1833 set_desc_base(ss, 0); /* flat segment */
1834 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1835 ss->g = 1; /* 4kb granularity */
1837 ss->type = 0x03; /* Read/Write, Accessed */
1838 ss->d = 1; /* 32bit stack segment */
1844 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1846 struct decode_cache *c = &ctxt->decode;
1847 struct desc_struct cs, ss;
1851 /* syscall is not available in real mode */
1852 if (ctxt->mode == X86EMUL_MODE_REAL ||
1853 ctxt->mode == X86EMUL_MODE_VM86)
1854 return emulate_ud(ctxt);
1856 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1858 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1860 cs_sel = (u16)(msr_data & 0xfffc);
1861 ss_sel = (u16)(msr_data + 8);
1863 if (is_long_mode(ctxt->vcpu)) {
1867 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1868 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1869 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1870 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1872 c->regs[VCPU_REGS_RCX] = c->eip;
1873 if (is_long_mode(ctxt->vcpu)) {
1874 #ifdef CONFIG_X86_64
1875 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1877 ops->get_msr(ctxt->vcpu,
1878 ctxt->mode == X86EMUL_MODE_PROT64 ?
1879 MSR_LSTAR : MSR_CSTAR, &msr_data);
1882 ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
1883 ctxt->eflags &= ~(msr_data | EFLG_RF);
1887 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1888 c->eip = (u32)msr_data;
1890 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1893 return X86EMUL_CONTINUE;
1897 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1899 struct decode_cache *c = &ctxt->decode;
1900 struct desc_struct cs, ss;
1904 /* inject #GP if in real mode */
1905 if (ctxt->mode == X86EMUL_MODE_REAL)
1906 return emulate_gp(ctxt, 0);
1908 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1909 * Therefore, we inject an #UD.
1911 if (ctxt->mode == X86EMUL_MODE_PROT64)
1912 return emulate_ud(ctxt);
1914 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1916 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1917 switch (ctxt->mode) {
1918 case X86EMUL_MODE_PROT32:
1919 if ((msr_data & 0xfffc) == 0x0)
1920 return emulate_gp(ctxt, 0);
1922 case X86EMUL_MODE_PROT64:
1923 if (msr_data == 0x0)
1924 return emulate_gp(ctxt, 0);
1928 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1929 cs_sel = (u16)msr_data;
1930 cs_sel &= ~SELECTOR_RPL_MASK;
1931 ss_sel = cs_sel + 8;
1932 ss_sel &= ~SELECTOR_RPL_MASK;
1933 if (ctxt->mode == X86EMUL_MODE_PROT64
1934 || is_long_mode(ctxt->vcpu)) {
1939 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1940 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1941 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1942 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1944 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
1947 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
1948 c->regs[VCPU_REGS_RSP] = msr_data;
1950 return X86EMUL_CONTINUE;
1954 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1956 struct decode_cache *c = &ctxt->decode;
1957 struct desc_struct cs, ss;
1962 /* inject #GP if in real mode or Virtual 8086 mode */
1963 if (ctxt->mode == X86EMUL_MODE_REAL ||
1964 ctxt->mode == X86EMUL_MODE_VM86)
1965 return emulate_gp(ctxt, 0);
1967 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1969 if ((c->rex_prefix & 0x8) != 0x0)
1970 usermode = X86EMUL_MODE_PROT64;
1972 usermode = X86EMUL_MODE_PROT32;
1976 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1978 case X86EMUL_MODE_PROT32:
1979 cs_sel = (u16)(msr_data + 16);
1980 if ((msr_data & 0xfffc) == 0x0)
1981 return emulate_gp(ctxt, 0);
1982 ss_sel = (u16)(msr_data + 24);
1984 case X86EMUL_MODE_PROT64:
1985 cs_sel = (u16)(msr_data + 32);
1986 if (msr_data == 0x0)
1987 return emulate_gp(ctxt, 0);
1988 ss_sel = cs_sel + 8;
1993 cs_sel |= SELECTOR_RPL_MASK;
1994 ss_sel |= SELECTOR_RPL_MASK;
1996 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1997 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1998 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1999 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
2001 c->eip = c->regs[VCPU_REGS_RDX];
2002 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
2004 return X86EMUL_CONTINUE;
2007 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
2008 struct x86_emulate_ops *ops)
2011 if (ctxt->mode == X86EMUL_MODE_REAL)
2013 if (ctxt->mode == X86EMUL_MODE_VM86)
2015 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2016 return ops->cpl(ctxt->vcpu) > iopl;
2019 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2020 struct x86_emulate_ops *ops,
2023 struct desc_struct tr_seg;
2026 u16 io_bitmap_ptr, perm, bit_idx = port & 0x7;
2027 unsigned mask = (1 << len) - 1;
2030 ops->get_cached_descriptor(&tr_seg, &base3, VCPU_SREG_TR, ctxt->vcpu);
2033 if (desc_limit_scaled(&tr_seg) < 103)
2035 base = get_desc_base(&tr_seg);
2036 #ifdef CONFIG_X86_64
2037 base |= ((u64)base3) << 32;
2039 r = ops->read_std(base + 102, &io_bitmap_ptr, 2, ctxt->vcpu, NULL);
2040 if (r != X86EMUL_CONTINUE)
2042 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2044 r = ops->read_std(base + io_bitmap_ptr + port/8, &perm, 2, ctxt->vcpu,
2046 if (r != X86EMUL_CONTINUE)
2048 if ((perm >> bit_idx) & mask)
2053 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2054 struct x86_emulate_ops *ops,
2060 if (emulator_bad_iopl(ctxt, ops))
2061 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
2064 ctxt->perm_ok = true;
2069 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2070 struct x86_emulate_ops *ops,
2071 struct tss_segment_16 *tss)
2073 struct decode_cache *c = &ctxt->decode;
2076 tss->flag = ctxt->eflags;
2077 tss->ax = c->regs[VCPU_REGS_RAX];
2078 tss->cx = c->regs[VCPU_REGS_RCX];
2079 tss->dx = c->regs[VCPU_REGS_RDX];
2080 tss->bx = c->regs[VCPU_REGS_RBX];
2081 tss->sp = c->regs[VCPU_REGS_RSP];
2082 tss->bp = c->regs[VCPU_REGS_RBP];
2083 tss->si = c->regs[VCPU_REGS_RSI];
2084 tss->di = c->regs[VCPU_REGS_RDI];
2086 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2087 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2088 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2089 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2090 tss->ldt = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2093 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2094 struct x86_emulate_ops *ops,
2095 struct tss_segment_16 *tss)
2097 struct decode_cache *c = &ctxt->decode;
2101 ctxt->eflags = tss->flag | 2;
2102 c->regs[VCPU_REGS_RAX] = tss->ax;
2103 c->regs[VCPU_REGS_RCX] = tss->cx;
2104 c->regs[VCPU_REGS_RDX] = tss->dx;
2105 c->regs[VCPU_REGS_RBX] = tss->bx;
2106 c->regs[VCPU_REGS_RSP] = tss->sp;
2107 c->regs[VCPU_REGS_RBP] = tss->bp;
2108 c->regs[VCPU_REGS_RSI] = tss->si;
2109 c->regs[VCPU_REGS_RDI] = tss->di;
2112 * SDM says that segment selectors are loaded before segment
2115 ops->set_segment_selector(tss->ldt, VCPU_SREG_LDTR, ctxt->vcpu);
2116 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2117 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2118 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2119 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2122 * Now load segment descriptors. If fault happenes at this stage
2123 * it is handled in a context of new task
2125 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
2126 if (ret != X86EMUL_CONTINUE)
2128 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2129 if (ret != X86EMUL_CONTINUE)
2131 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2132 if (ret != X86EMUL_CONTINUE)
2134 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2135 if (ret != X86EMUL_CONTINUE)
2137 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2138 if (ret != X86EMUL_CONTINUE)
2141 return X86EMUL_CONTINUE;
2144 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2145 struct x86_emulate_ops *ops,
2146 u16 tss_selector, u16 old_tss_sel,
2147 ulong old_tss_base, struct desc_struct *new_desc)
2149 struct tss_segment_16 tss_seg;
2151 u32 new_tss_base = get_desc_base(new_desc);
2153 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2155 if (ret != X86EMUL_CONTINUE)
2156 /* FIXME: need to provide precise fault address */
2159 save_state_to_tss16(ctxt, ops, &tss_seg);
2161 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2163 if (ret != X86EMUL_CONTINUE)
2164 /* FIXME: need to provide precise fault address */
2167 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2169 if (ret != X86EMUL_CONTINUE)
2170 /* FIXME: need to provide precise fault address */
2173 if (old_tss_sel != 0xffff) {
2174 tss_seg.prev_task_link = old_tss_sel;
2176 ret = ops->write_std(new_tss_base,
2177 &tss_seg.prev_task_link,
2178 sizeof tss_seg.prev_task_link,
2179 ctxt->vcpu, &ctxt->exception);
2180 if (ret != X86EMUL_CONTINUE)
2181 /* FIXME: need to provide precise fault address */
2185 return load_state_from_tss16(ctxt, ops, &tss_seg);
2188 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2189 struct x86_emulate_ops *ops,
2190 struct tss_segment_32 *tss)
2192 struct decode_cache *c = &ctxt->decode;
2194 tss->cr3 = ops->get_cr(3, ctxt->vcpu);
2196 tss->eflags = ctxt->eflags;
2197 tss->eax = c->regs[VCPU_REGS_RAX];
2198 tss->ecx = c->regs[VCPU_REGS_RCX];
2199 tss->edx = c->regs[VCPU_REGS_RDX];
2200 tss->ebx = c->regs[VCPU_REGS_RBX];
2201 tss->esp = c->regs[VCPU_REGS_RSP];
2202 tss->ebp = c->regs[VCPU_REGS_RBP];
2203 tss->esi = c->regs[VCPU_REGS_RSI];
2204 tss->edi = c->regs[VCPU_REGS_RDI];
2206 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2207 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2208 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2209 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2210 tss->fs = ops->get_segment_selector(VCPU_SREG_FS, ctxt->vcpu);
2211 tss->gs = ops->get_segment_selector(VCPU_SREG_GS, ctxt->vcpu);
2212 tss->ldt_selector = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2215 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2216 struct x86_emulate_ops *ops,
2217 struct tss_segment_32 *tss)
2219 struct decode_cache *c = &ctxt->decode;
2222 if (ops->set_cr(3, tss->cr3, ctxt->vcpu))
2223 return emulate_gp(ctxt, 0);
2225 ctxt->eflags = tss->eflags | 2;
2226 c->regs[VCPU_REGS_RAX] = tss->eax;
2227 c->regs[VCPU_REGS_RCX] = tss->ecx;
2228 c->regs[VCPU_REGS_RDX] = tss->edx;
2229 c->regs[VCPU_REGS_RBX] = tss->ebx;
2230 c->regs[VCPU_REGS_RSP] = tss->esp;
2231 c->regs[VCPU_REGS_RBP] = tss->ebp;
2232 c->regs[VCPU_REGS_RSI] = tss->esi;
2233 c->regs[VCPU_REGS_RDI] = tss->edi;
2236 * SDM says that segment selectors are loaded before segment
2239 ops->set_segment_selector(tss->ldt_selector, VCPU_SREG_LDTR, ctxt->vcpu);
2240 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2241 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2242 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2243 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2244 ops->set_segment_selector(tss->fs, VCPU_SREG_FS, ctxt->vcpu);
2245 ops->set_segment_selector(tss->gs, VCPU_SREG_GS, ctxt->vcpu);
2248 * Now load segment descriptors. If fault happenes at this stage
2249 * it is handled in a context of new task
2251 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2252 if (ret != X86EMUL_CONTINUE)
2254 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2255 if (ret != X86EMUL_CONTINUE)
2257 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2258 if (ret != X86EMUL_CONTINUE)
2260 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2261 if (ret != X86EMUL_CONTINUE)
2263 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2264 if (ret != X86EMUL_CONTINUE)
2266 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2267 if (ret != X86EMUL_CONTINUE)
2269 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2270 if (ret != X86EMUL_CONTINUE)
2273 return X86EMUL_CONTINUE;
2276 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2277 struct x86_emulate_ops *ops,
2278 u16 tss_selector, u16 old_tss_sel,
2279 ulong old_tss_base, struct desc_struct *new_desc)
2281 struct tss_segment_32 tss_seg;
2283 u32 new_tss_base = get_desc_base(new_desc);
2285 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2287 if (ret != X86EMUL_CONTINUE)
2288 /* FIXME: need to provide precise fault address */
2291 save_state_to_tss32(ctxt, ops, &tss_seg);
2293 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2295 if (ret != X86EMUL_CONTINUE)
2296 /* FIXME: need to provide precise fault address */
2299 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2301 if (ret != X86EMUL_CONTINUE)
2302 /* FIXME: need to provide precise fault address */
2305 if (old_tss_sel != 0xffff) {
2306 tss_seg.prev_task_link = old_tss_sel;
2308 ret = ops->write_std(new_tss_base,
2309 &tss_seg.prev_task_link,
2310 sizeof tss_seg.prev_task_link,
2311 ctxt->vcpu, &ctxt->exception);
2312 if (ret != X86EMUL_CONTINUE)
2313 /* FIXME: need to provide precise fault address */
2317 return load_state_from_tss32(ctxt, ops, &tss_seg);
2320 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2321 struct x86_emulate_ops *ops,
2322 u16 tss_selector, int reason,
2323 bool has_error_code, u32 error_code)
2325 struct desc_struct curr_tss_desc, next_tss_desc;
2327 u16 old_tss_sel = ops->get_segment_selector(VCPU_SREG_TR, ctxt->vcpu);
2328 ulong old_tss_base =
2329 ops->get_cached_segment_base(VCPU_SREG_TR, ctxt->vcpu);
2332 /* FIXME: old_tss_base == ~0 ? */
2334 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2335 if (ret != X86EMUL_CONTINUE)
2337 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2338 if (ret != X86EMUL_CONTINUE)
2341 /* FIXME: check that next_tss_desc is tss */
2343 if (reason != TASK_SWITCH_IRET) {
2344 if ((tss_selector & 3) > next_tss_desc.dpl ||
2345 ops->cpl(ctxt->vcpu) > next_tss_desc.dpl)
2346 return emulate_gp(ctxt, 0);
2349 desc_limit = desc_limit_scaled(&next_tss_desc);
2350 if (!next_tss_desc.p ||
2351 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2352 desc_limit < 0x2b)) {
2353 emulate_ts(ctxt, tss_selector & 0xfffc);
2354 return X86EMUL_PROPAGATE_FAULT;
2357 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2358 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2359 write_segment_descriptor(ctxt, ops, old_tss_sel,
2363 if (reason == TASK_SWITCH_IRET)
2364 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2366 /* set back link to prev task only if NT bit is set in eflags
2367 note that old_tss_sel is not used afetr this point */
2368 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2369 old_tss_sel = 0xffff;
2371 if (next_tss_desc.type & 8)
2372 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2373 old_tss_base, &next_tss_desc);
2375 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2376 old_tss_base, &next_tss_desc);
2377 if (ret != X86EMUL_CONTINUE)
2380 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2381 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2383 if (reason != TASK_SWITCH_IRET) {
2384 next_tss_desc.type |= (1 << 1); /* set busy flag */
2385 write_segment_descriptor(ctxt, ops, tss_selector,
2389 ops->set_cr(0, ops->get_cr(0, ctxt->vcpu) | X86_CR0_TS, ctxt->vcpu);
2390 ops->set_cached_descriptor(&next_tss_desc, 0, VCPU_SREG_TR, ctxt->vcpu);
2391 ops->set_segment_selector(tss_selector, VCPU_SREG_TR, ctxt->vcpu);
2393 if (has_error_code) {
2394 struct decode_cache *c = &ctxt->decode;
2396 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2398 c->src.val = (unsigned long) error_code;
2399 ret = em_push(ctxt);
2405 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2406 u16 tss_selector, int reason,
2407 bool has_error_code, u32 error_code)
2409 struct x86_emulate_ops *ops = ctxt->ops;
2410 struct decode_cache *c = &ctxt->decode;
2414 c->dst.type = OP_NONE;
2416 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2417 has_error_code, error_code);
2419 if (rc == X86EMUL_CONTINUE)
2422 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2425 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2426 int reg, struct operand *op)
2428 struct decode_cache *c = &ctxt->decode;
2429 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2431 register_address_increment(c, &c->regs[reg], df * op->bytes);
2432 op->addr.mem.ea = register_address(c, c->regs[reg]);
2433 op->addr.mem.seg = seg;
2436 static int em_das(struct x86_emulate_ctxt *ctxt)
2438 struct decode_cache *c = &ctxt->decode;
2440 bool af, cf, old_cf;
2442 cf = ctxt->eflags & X86_EFLAGS_CF;
2448 af = ctxt->eflags & X86_EFLAGS_AF;
2449 if ((al & 0x0f) > 9 || af) {
2451 cf = old_cf | (al >= 250);
2456 if (old_al > 0x99 || old_cf) {
2462 /* Set PF, ZF, SF */
2463 c->src.type = OP_IMM;
2466 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2467 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2469 ctxt->eflags |= X86_EFLAGS_CF;
2471 ctxt->eflags |= X86_EFLAGS_AF;
2472 return X86EMUL_CONTINUE;
2475 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2477 struct decode_cache *c = &ctxt->decode;
2482 old_cs = ctxt->ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2485 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
2486 if (load_segment_descriptor(ctxt, ctxt->ops, sel, VCPU_SREG_CS))
2487 return X86EMUL_CONTINUE;
2490 memcpy(&c->eip, c->src.valptr, c->op_bytes);
2492 c->src.val = old_cs;
2494 if (rc != X86EMUL_CONTINUE)
2497 c->src.val = old_eip;
2498 return em_push(ctxt);
2501 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2503 struct decode_cache *c = &ctxt->decode;
2506 c->dst.type = OP_REG;
2507 c->dst.addr.reg = &c->eip;
2508 c->dst.bytes = c->op_bytes;
2509 rc = emulate_pop(ctxt, ctxt->ops, &c->dst.val, c->op_bytes);
2510 if (rc != X86EMUL_CONTINUE)
2512 register_address_increment(c, &c->regs[VCPU_REGS_RSP], c->src.val);
2513 return X86EMUL_CONTINUE;
2516 static int em_imul(struct x86_emulate_ctxt *ctxt)
2518 struct decode_cache *c = &ctxt->decode;
2520 emulate_2op_SrcV_nobyte("imul", c->src, c->dst, ctxt->eflags);
2521 return X86EMUL_CONTINUE;
2524 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2526 struct decode_cache *c = &ctxt->decode;
2528 c->dst.val = c->src2.val;
2529 return em_imul(ctxt);
2532 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2534 struct decode_cache *c = &ctxt->decode;
2536 c->dst.type = OP_REG;
2537 c->dst.bytes = c->src.bytes;
2538 c->dst.addr.reg = &c->regs[VCPU_REGS_RDX];
2539 c->dst.val = ~((c->src.val >> (c->src.bytes * 8 - 1)) - 1);
2541 return X86EMUL_CONTINUE;
2544 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2546 struct decode_cache *c = &ctxt->decode;
2549 ctxt->ops->get_msr(ctxt->vcpu, MSR_IA32_TSC, &tsc);
2550 c->regs[VCPU_REGS_RAX] = (u32)tsc;
2551 c->regs[VCPU_REGS_RDX] = tsc >> 32;
2552 return X86EMUL_CONTINUE;
2555 static int em_mov(struct x86_emulate_ctxt *ctxt)
2557 struct decode_cache *c = &ctxt->decode;
2558 c->dst.val = c->src.val;
2559 return X86EMUL_CONTINUE;
2562 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2564 struct decode_cache *c = &ctxt->decode;
2565 memcpy(&c->dst.vec_val, &c->src.vec_val, c->op_bytes);
2566 return X86EMUL_CONTINUE;
2569 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2571 struct decode_cache *c = &ctxt->decode;
2575 rc = linearize(ctxt, c->src.addr.mem, 1, false, &linear);
2576 if (rc == X86EMUL_CONTINUE)
2577 emulate_invlpg(ctxt->vcpu, linear);
2578 /* Disable writeback. */
2579 c->dst.type = OP_NONE;
2580 return X86EMUL_CONTINUE;
2583 static bool valid_cr(int nr)
2595 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2597 struct decode_cache *c = &ctxt->decode;
2599 if (!valid_cr(c->modrm_reg))
2600 return emulate_ud(ctxt);
2602 return X86EMUL_CONTINUE;
2605 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2607 struct decode_cache *c = &ctxt->decode;
2608 u64 new_val = c->src.val64;
2609 int cr = c->modrm_reg;
2611 static u64 cr_reserved_bits[] = {
2612 0xffffffff00000000ULL,
2613 0, 0, 0, /* CR3 checked later */
2620 return emulate_ud(ctxt);
2622 if (new_val & cr_reserved_bits[cr])
2623 return emulate_gp(ctxt, 0);
2628 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2629 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2630 return emulate_gp(ctxt, 0);
2632 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2633 ctxt->ops->get_msr(ctxt->vcpu, MSR_EFER, &efer);
2635 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2636 !(cr4 & X86_CR4_PAE))
2637 return emulate_gp(ctxt, 0);
2644 if (is_long_mode(ctxt->vcpu))
2645 rsvd = CR3_L_MODE_RESERVED_BITS;
2646 else if (is_pae(ctxt->vcpu))
2647 rsvd = CR3_PAE_RESERVED_BITS;
2648 else if (is_paging(ctxt->vcpu))
2649 rsvd = CR3_NONPAE_RESERVED_BITS;
2652 return emulate_gp(ctxt, 0);
2659 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2660 ctxt->ops->get_msr(ctxt->vcpu, MSR_EFER, &efer);
2662 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2663 return emulate_gp(ctxt, 0);
2669 return X86EMUL_CONTINUE;
2672 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2676 ctxt->ops->get_dr(7, &dr7, ctxt->vcpu);
2678 /* Check if DR7.Global_Enable is set */
2679 return dr7 & (1 << 13);
2682 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2684 struct decode_cache *c = &ctxt->decode;
2685 int dr = c->modrm_reg;
2689 return emulate_ud(ctxt);
2691 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2692 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2693 return emulate_ud(ctxt);
2695 if (check_dr7_gd(ctxt))
2696 return emulate_db(ctxt);
2698 return X86EMUL_CONTINUE;
2701 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2703 struct decode_cache *c = &ctxt->decode;
2704 u64 new_val = c->src.val64;
2705 int dr = c->modrm_reg;
2707 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2708 return emulate_gp(ctxt, 0);
2710 return check_dr_read(ctxt);
2713 static int check_svme(struct x86_emulate_ctxt *ctxt)
2717 ctxt->ops->get_msr(ctxt->vcpu, MSR_EFER, &efer);
2719 if (!(efer & EFER_SVME))
2720 return emulate_ud(ctxt);
2722 return X86EMUL_CONTINUE;
2725 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
2727 u64 rax = kvm_register_read(ctxt->vcpu, VCPU_REGS_RAX);
2729 /* Valid physical address? */
2730 if (rax & 0xffff000000000000)
2731 return emulate_gp(ctxt, 0);
2733 return check_svme(ctxt);
2736 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
2738 u64 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2740 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt->vcpu))
2741 return emulate_ud(ctxt);
2743 return X86EMUL_CONTINUE;
2746 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
2748 u64 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2749 u64 rcx = kvm_register_read(ctxt->vcpu, VCPU_REGS_RCX);
2751 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt->vcpu)) ||
2753 return emulate_gp(ctxt, 0);
2755 return X86EMUL_CONTINUE;
2758 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
2760 struct decode_cache *c = &ctxt->decode;
2762 c->dst.bytes = min(c->dst.bytes, 4u);
2763 if (!emulator_io_permited(ctxt, ctxt->ops, c->src.val, c->dst.bytes))
2764 return emulate_gp(ctxt, 0);
2766 return X86EMUL_CONTINUE;
2769 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
2771 struct decode_cache *c = &ctxt->decode;
2773 c->src.bytes = min(c->src.bytes, 4u);
2774 if (!emulator_io_permited(ctxt, ctxt->ops, c->dst.val, c->src.bytes))
2775 return emulate_gp(ctxt, 0);
2777 return X86EMUL_CONTINUE;
2780 #define D(_y) { .flags = (_y) }
2781 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2782 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
2783 .check_perm = (_p) }
2785 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
2786 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2787 #define GD(_f, _g) { .flags = ((_f) | Group | GroupDual), .u.gdual = (_g) }
2788 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2789 #define II(_f, _e, _i) \
2790 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2791 #define IIP(_f, _e, _i, _p) \
2792 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
2793 .check_perm = (_p) }
2794 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
2796 #define D2bv(_f) D((_f) | ByteOp), D(_f)
2797 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
2798 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
2800 #define D6ALU(_f) D2bv((_f) | DstMem | SrcReg | ModRM), \
2801 D2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock), \
2802 D2bv(((_f) & ~Lock) | DstAcc | SrcImm)
2804 static struct opcode group7_rm1[] = {
2805 DI(SrcNone | ModRM | Priv, monitor),
2806 DI(SrcNone | ModRM | Priv, mwait),
2810 static struct opcode group7_rm3[] = {
2811 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
2812 DI(SrcNone | ModRM | Prot | VendorSpecific, vmmcall),
2813 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
2814 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
2815 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
2816 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
2817 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
2818 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
2821 static struct opcode group7_rm7[] = {
2823 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
2826 static struct opcode group1[] = {
2830 static struct opcode group1A[] = {
2831 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
2834 static struct opcode group3[] = {
2835 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
2836 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
2837 X4(D(SrcMem | ModRM)),
2840 static struct opcode group4[] = {
2841 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
2845 static struct opcode group5[] = {
2846 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
2847 D(SrcMem | ModRM | Stack),
2848 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
2849 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
2850 D(SrcMem | ModRM | Stack), N,
2853 static struct opcode group6[] = {
2854 DI(ModRM | Prot, sldt),
2855 DI(ModRM | Prot, str),
2856 DI(ModRM | Prot | Priv, lldt),
2857 DI(ModRM | Prot | Priv, ltr),
2861 static struct group_dual group7 = { {
2862 DI(ModRM | Mov | DstMem | Priv, sgdt),
2863 DI(ModRM | Mov | DstMem | Priv, sidt),
2864 DI(ModRM | SrcMem | Priv, lgdt), DI(ModRM | SrcMem | Priv, lidt),
2865 DI(SrcNone | ModRM | DstMem | Mov, smsw), N,
2866 DI(SrcMem16 | ModRM | Mov | Priv, lmsw),
2867 DI(SrcMem | ModRM | ByteOp | Priv | NoAccess, invlpg),
2869 D(SrcNone | ModRM | Priv | VendorSpecific), EXT(0, group7_rm1),
2870 N, EXT(0, group7_rm3),
2871 DI(SrcNone | ModRM | DstMem | Mov, smsw), N,
2872 DI(SrcMem16 | ModRM | Mov | Priv, lmsw), EXT(0, group7_rm7),
2875 static struct opcode group8[] = {
2877 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
2878 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
2881 static struct group_dual group9 = { {
2882 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
2884 N, N, N, N, N, N, N, N,
2887 static struct opcode group11[] = {
2888 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
2891 static struct gprefix pfx_0f_6f_0f_7f = {
2892 N, N, N, I(Sse, em_movdqu),
2895 static struct opcode opcode_table[256] = {
2898 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2901 D(ImplicitOps | Stack | No64), N,
2904 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2907 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2911 D6ALU(Lock), N, I(ByteOp | DstAcc | No64, em_das),
2919 X8(I(SrcReg | Stack, em_push)),
2921 X8(D(DstReg | Stack)),
2923 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2924 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
2927 I(SrcImm | Mov | Stack, em_push),
2928 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
2929 I(SrcImmByte | Mov | Stack, em_push),
2930 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
2931 D2bvIP(DstDI | Mov | String, ins, check_perm_in), /* insb, insw/insd */
2932 D2bvIP(SrcSI | ImplicitOps | String, outs, check_perm_out), /* outsb, outsw/outsd */
2936 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
2937 G(DstMem | SrcImm | ModRM | Group, group1),
2938 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
2939 G(DstMem | SrcImmByte | ModRM | Group, group1),
2940 D2bv(DstMem | SrcReg | ModRM), D2bv(DstMem | SrcReg | ModRM | Lock),
2942 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
2943 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
2944 D(DstMem | SrcNone | ModRM | Mov), D(ModRM | SrcMem | NoAccess | DstReg),
2945 D(ImplicitOps | SrcMem16 | ModRM), G(0, group1A),
2947 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
2949 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
2950 I(SrcImmFAddr | No64, em_call_far), N,
2951 DI(ImplicitOps | Stack, pushf), DI(ImplicitOps | Stack, popf), N, N,
2953 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
2954 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
2955 I2bv(SrcSI | DstDI | Mov | String, em_mov),
2956 D2bv(SrcSI | DstDI | String),
2958 D2bv(DstAcc | SrcImm),
2959 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
2960 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
2961 D2bv(SrcAcc | DstDI | String),
2963 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
2965 X8(I(DstReg | SrcImm | Mov, em_mov)),
2967 D2bv(DstMem | SrcImmByte | ModRM),
2968 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
2969 D(ImplicitOps | Stack),
2970 D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
2971 G(ByteOp, group11), G(0, group11),
2973 N, N, N, D(ImplicitOps | Stack),
2974 D(ImplicitOps), DI(SrcImmByte, intn),
2975 D(ImplicitOps | No64), DI(ImplicitOps, iret),
2977 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
2980 N, N, N, N, N, N, N, N,
2983 D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
2984 D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
2986 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
2987 D(SrcImmFAddr | No64), D(SrcImmByte | ImplicitOps),
2988 D2bvIP(SrcNone | DstAcc, in, check_perm_in),
2989 D2bvIP(SrcAcc | ImplicitOps, out, check_perm_out),
2991 N, DI(ImplicitOps, icebp), N, N,
2992 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
2993 G(ByteOp, group3), G(0, group3),
2995 D(ImplicitOps), D(ImplicitOps), D(ImplicitOps), D(ImplicitOps),
2996 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
2999 static struct opcode twobyte_table[256] = {
3001 G(0, group6), GD(0, &group7), N, N,
3002 N, D(ImplicitOps | VendorSpecific), DI(ImplicitOps | Priv, clts), N,
3003 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3004 N, D(ImplicitOps | ModRM), N, N,
3006 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3008 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3009 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3010 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3011 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3013 N, N, N, N, N, N, N, N,
3015 DI(ImplicitOps | Priv, wrmsr),
3016 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3017 DI(ImplicitOps | Priv, rdmsr),
3018 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3019 D(ImplicitOps | VendorSpecific), D(ImplicitOps | Priv | VendorSpecific),
3021 N, N, N, N, N, N, N, N,
3023 X16(D(DstReg | SrcMem | ModRM | Mov)),
3025 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3030 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3035 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3039 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3041 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3042 DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3043 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3044 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3046 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3047 DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3048 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3049 D(DstMem | SrcReg | Src2CL | ModRM),
3050 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3052 D2bv(DstMem | SrcReg | ModRM | Lock),
3053 D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3054 D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
3055 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3058 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3059 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3060 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3062 D2bv(DstMem | SrcReg | ModRM | Lock),
3063 N, D(DstMem | SrcReg | ModRM | Mov),
3064 N, N, N, GD(0, &group9),
3065 N, N, N, N, N, N, N, N,
3067 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3069 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3071 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3087 static unsigned imm_size(struct decode_cache *c)
3091 size = (c->d & ByteOp) ? 1 : c->op_bytes;
3097 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3098 unsigned size, bool sign_extension)
3100 struct decode_cache *c = &ctxt->decode;
3101 struct x86_emulate_ops *ops = ctxt->ops;
3102 int rc = X86EMUL_CONTINUE;
3106 op->addr.mem.ea = c->eip;
3107 /* NB. Immediates are sign-extended as necessary. */
3108 switch (op->bytes) {
3110 op->val = insn_fetch(s8, 1, c->eip);
3113 op->val = insn_fetch(s16, 2, c->eip);
3116 op->val = insn_fetch(s32, 4, c->eip);
3119 if (!sign_extension) {
3120 switch (op->bytes) {
3128 op->val &= 0xffffffff;
3137 x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3139 struct x86_emulate_ops *ops = ctxt->ops;
3140 struct decode_cache *c = &ctxt->decode;
3141 int rc = X86EMUL_CONTINUE;
3142 int mode = ctxt->mode;
3143 int def_op_bytes, def_ad_bytes, dual, goffset, simd_prefix;
3144 bool op_prefix = false;
3145 struct opcode opcode, *g_mod012, *g_mod3;
3146 struct operand memop = { .type = OP_NONE };
3149 c->fetch.start = c->eip;
3150 c->fetch.end = c->fetch.start + insn_len;
3152 memcpy(c->fetch.data, insn, insn_len);
3155 case X86EMUL_MODE_REAL:
3156 case X86EMUL_MODE_VM86:
3157 case X86EMUL_MODE_PROT16:
3158 def_op_bytes = def_ad_bytes = 2;
3160 case X86EMUL_MODE_PROT32:
3161 def_op_bytes = def_ad_bytes = 4;
3163 #ifdef CONFIG_X86_64
3164 case X86EMUL_MODE_PROT64:
3173 c->op_bytes = def_op_bytes;
3174 c->ad_bytes = def_ad_bytes;
3176 /* Legacy prefixes. */
3178 switch (c->b = insn_fetch(u8, 1, c->eip)) {
3179 case 0x66: /* operand-size override */
3181 /* switch between 2/4 bytes */
3182 c->op_bytes = def_op_bytes ^ 6;
3184 case 0x67: /* address-size override */
3185 if (mode == X86EMUL_MODE_PROT64)
3186 /* switch between 4/8 bytes */
3187 c->ad_bytes = def_ad_bytes ^ 12;
3189 /* switch between 2/4 bytes */
3190 c->ad_bytes = def_ad_bytes ^ 6;
3192 case 0x26: /* ES override */
3193 case 0x2e: /* CS override */
3194 case 0x36: /* SS override */
3195 case 0x3e: /* DS override */
3196 set_seg_override(c, (c->b >> 3) & 3);
3198 case 0x64: /* FS override */
3199 case 0x65: /* GS override */
3200 set_seg_override(c, c->b & 7);
3202 case 0x40 ... 0x4f: /* REX */
3203 if (mode != X86EMUL_MODE_PROT64)
3205 c->rex_prefix = c->b;
3207 case 0xf0: /* LOCK */
3210 case 0xf2: /* REPNE/REPNZ */
3211 case 0xf3: /* REP/REPE/REPZ */
3212 c->rep_prefix = c->b;
3218 /* Any legacy prefix after a REX prefix nullifies its effect. */
3226 if (c->rex_prefix & 8)
3227 c->op_bytes = 8; /* REX.W */
3229 /* Opcode byte(s). */
3230 opcode = opcode_table[c->b];
3231 /* Two-byte opcode? */
3234 c->b = insn_fetch(u8, 1, c->eip);
3235 opcode = twobyte_table[c->b];
3237 c->d = opcode.flags;
3240 dual = c->d & GroupDual;
3241 c->modrm = insn_fetch(u8, 1, c->eip);
3244 if (c->d & GroupDual) {
3245 g_mod012 = opcode.u.gdual->mod012;
3246 g_mod3 = opcode.u.gdual->mod3;
3248 g_mod012 = g_mod3 = opcode.u.group;
3250 c->d &= ~(Group | GroupDual);
3252 goffset = (c->modrm >> 3) & 7;
3254 if ((c->modrm >> 6) == 3)
3255 opcode = g_mod3[goffset];
3257 opcode = g_mod012[goffset];
3259 if (opcode.flags & RMExt) {
3260 goffset = c->modrm & 7;
3261 opcode = opcode.u.group[goffset];
3264 c->d |= opcode.flags;
3267 if (c->d & Prefix) {
3268 if (c->rep_prefix && op_prefix)
3269 return X86EMUL_UNHANDLEABLE;
3270 simd_prefix = op_prefix ? 0x66 : c->rep_prefix;
3271 switch (simd_prefix) {
3272 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3273 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3274 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3275 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3277 c->d |= opcode.flags;
3280 c->execute = opcode.u.execute;
3281 c->check_perm = opcode.check_perm;
3282 c->intercept = opcode.intercept;
3285 if (c->d == 0 || (c->d & Undefined))
3288 if (!(c->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3291 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
3294 if (c->d & Op3264) {
3295 if (mode == X86EMUL_MODE_PROT64)
3304 /* ModRM and SIB bytes. */
3306 rc = decode_modrm(ctxt, ops, &memop);
3307 if (!c->has_seg_override)
3308 set_seg_override(c, c->modrm_seg);
3309 } else if (c->d & MemAbs)
3310 rc = decode_abs(ctxt, ops, &memop);
3311 if (rc != X86EMUL_CONTINUE)
3314 if (!c->has_seg_override)
3315 set_seg_override(c, VCPU_SREG_DS);
3317 memop.addr.mem.seg = seg_override(ctxt, ops, c);
3319 if (memop.type == OP_MEM && c->ad_bytes != 8)
3320 memop.addr.mem.ea = (u32)memop.addr.mem.ea;
3322 if (memop.type == OP_MEM && c->rip_relative)
3323 memop.addr.mem.ea += c->eip;
3326 * Decode and fetch the source operand: register, memory
3329 switch (c->d & SrcMask) {
3333 decode_register_operand(ctxt, &c->src, c, 0);
3342 memop.bytes = (c->d & ByteOp) ? 1 :
3348 rc = decode_imm(ctxt, &c->src, 2, false);
3351 rc = decode_imm(ctxt, &c->src, imm_size(c), true);
3354 rc = decode_imm(ctxt, &c->src, imm_size(c), false);
3357 rc = decode_imm(ctxt, &c->src, 1, true);
3360 rc = decode_imm(ctxt, &c->src, 1, false);
3363 c->src.type = OP_REG;
3364 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3365 c->src.addr.reg = &c->regs[VCPU_REGS_RAX];
3366 fetch_register_operand(&c->src);
3373 c->src.type = OP_MEM;
3374 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3375 c->src.addr.mem.ea =
3376 register_address(c, c->regs[VCPU_REGS_RSI]);
3377 c->src.addr.mem.seg = seg_override(ctxt, ops, c),
3381 c->src.type = OP_IMM;
3382 c->src.addr.mem.ea = c->eip;
3383 c->src.bytes = c->op_bytes + 2;
3384 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
3387 memop.bytes = c->op_bytes + 2;
3392 if (rc != X86EMUL_CONTINUE)
3396 * Decode and fetch the second source operand: register, memory
3399 switch (c->d & Src2Mask) {
3404 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
3407 rc = decode_imm(ctxt, &c->src2, 1, true);
3414 rc = decode_imm(ctxt, &c->src2, imm_size(c), true);
3418 if (rc != X86EMUL_CONTINUE)
3421 /* Decode and fetch the destination operand: register or memory. */
3422 switch (c->d & DstMask) {
3424 decode_register_operand(ctxt, &c->dst, c,
3425 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
3428 c->dst.type = OP_IMM;
3429 c->dst.addr.mem.ea = c->eip;
3431 c->dst.val = insn_fetch(u8, 1, c->eip);
3436 if ((c->d & DstMask) == DstMem64)
3439 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3441 fetch_bit_operand(c);
3442 c->dst.orig_val = c->dst.val;
3445 c->dst.type = OP_REG;
3446 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3447 c->dst.addr.reg = &c->regs[VCPU_REGS_RAX];
3448 fetch_register_operand(&c->dst);
3449 c->dst.orig_val = c->dst.val;
3452 c->dst.type = OP_MEM;
3453 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3454 c->dst.addr.mem.ea =
3455 register_address(c, c->regs[VCPU_REGS_RDI]);
3456 c->dst.addr.mem.seg = VCPU_SREG_ES;
3460 /* Special instructions do their own operand decoding. */
3462 c->dst.type = OP_NONE; /* Disable writeback. */
3467 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3470 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3472 struct decode_cache *c = &ctxt->decode;
3474 /* The second termination condition only applies for REPE
3475 * and REPNE. Test if the repeat string operation prefix is
3476 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3477 * corresponding termination condition according to:
3478 * - if REPE/REPZ and ZF = 0 then done
3479 * - if REPNE/REPNZ and ZF = 1 then done
3481 if (((c->b == 0xa6) || (c->b == 0xa7) ||
3482 (c->b == 0xae) || (c->b == 0xaf))
3483 && (((c->rep_prefix == REPE_PREFIX) &&
3484 ((ctxt->eflags & EFLG_ZF) == 0))
3485 || ((c->rep_prefix == REPNE_PREFIX) &&
3486 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3493 x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3495 struct x86_emulate_ops *ops = ctxt->ops;
3497 struct decode_cache *c = &ctxt->decode;
3498 int rc = X86EMUL_CONTINUE;
3499 int saved_dst_type = c->dst.type;
3500 int irq; /* Used for int 3, int, and into */
3502 ctxt->decode.mem_read.pos = 0;
3504 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
3505 rc = emulate_ud(ctxt);
3509 /* LOCK prefix is allowed only with some instructions */
3510 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
3511 rc = emulate_ud(ctxt);
3515 if ((c->d & SrcMask) == SrcMemFAddr && c->src.type != OP_MEM) {
3516 rc = emulate_ud(ctxt);
3521 && ((ops->get_cr(0, ctxt->vcpu) & X86_CR0_EM)
3522 || !(ops->get_cr(4, ctxt->vcpu) & X86_CR4_OSFXSR))) {
3523 rc = emulate_ud(ctxt);
3527 if ((c->d & Sse) && (ops->get_cr(0, ctxt->vcpu) & X86_CR0_TS)) {
3528 rc = emulate_nm(ctxt);
3532 if (unlikely(ctxt->guest_mode) && c->intercept) {
3533 rc = emulator_check_intercept(ctxt, c->intercept,
3534 X86_ICPT_PRE_EXCEPT);
3535 if (rc != X86EMUL_CONTINUE)
3539 /* Privileged instruction can be executed only in CPL=0 */
3540 if ((c->d & Priv) && ops->cpl(ctxt->vcpu)) {
3541 rc = emulate_gp(ctxt, 0);
3545 /* Instruction can only be executed in protected mode */
3546 if ((c->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3547 rc = emulate_ud(ctxt);
3551 /* Do instruction specific permission checks */
3552 if (c->check_perm) {
3553 rc = c->check_perm(ctxt);
3554 if (rc != X86EMUL_CONTINUE)
3558 if (unlikely(ctxt->guest_mode) && c->intercept) {
3559 rc = emulator_check_intercept(ctxt, c->intercept,
3560 X86_ICPT_POST_EXCEPT);
3561 if (rc != X86EMUL_CONTINUE)
3565 if (c->rep_prefix && (c->d & String)) {
3566 /* All REP prefixes have the same first termination condition */
3567 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
3573 if ((c->src.type == OP_MEM) && !(c->d & NoAccess)) {
3574 rc = segmented_read(ctxt, c->src.addr.mem,
3575 c->src.valptr, c->src.bytes);
3576 if (rc != X86EMUL_CONTINUE)
3578 c->src.orig_val64 = c->src.val64;
3581 if (c->src2.type == OP_MEM) {
3582 rc = segmented_read(ctxt, c->src2.addr.mem,
3583 &c->src2.val, c->src2.bytes);
3584 if (rc != X86EMUL_CONTINUE)
3588 if ((c->d & DstMask) == ImplicitOps)
3592 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
3593 /* optimisation - avoid slow emulated read if Mov */
3594 rc = segmented_read(ctxt, c->dst.addr.mem,
3595 &c->dst.val, c->dst.bytes);
3596 if (rc != X86EMUL_CONTINUE)
3599 c->dst.orig_val = c->dst.val;
3603 if (unlikely(ctxt->guest_mode) && c->intercept) {
3604 rc = emulator_check_intercept(ctxt, c->intercept,
3605 X86_ICPT_POST_MEMACCESS);
3606 if (rc != X86EMUL_CONTINUE)
3611 rc = c->execute(ctxt);
3612 if (rc != X86EMUL_CONTINUE)
3623 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
3625 case 0x06: /* push es */
3626 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
3628 case 0x07: /* pop es */
3629 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
3633 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
3635 case 0x0e: /* push cs */
3636 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
3640 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
3642 case 0x16: /* push ss */
3643 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
3645 case 0x17: /* pop ss */
3646 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
3650 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
3652 case 0x1e: /* push ds */
3653 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
3655 case 0x1f: /* pop ds */
3656 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
3660 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
3664 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
3668 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
3672 c->dst.type = OP_NONE; /* Disable writeback. */
3673 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
3675 case 0x40 ... 0x47: /* inc r16/r32 */
3676 emulate_1op("inc", c->dst, ctxt->eflags);
3678 case 0x48 ... 0x4f: /* dec r16/r32 */
3679 emulate_1op("dec", c->dst, ctxt->eflags);
3681 case 0x58 ... 0x5f: /* pop reg */
3683 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
3685 case 0x60: /* pusha */
3686 rc = emulate_pusha(ctxt);
3688 case 0x61: /* popa */
3689 rc = emulate_popa(ctxt, ops);
3691 case 0x63: /* movsxd */
3692 if (ctxt->mode != X86EMUL_MODE_PROT64)
3693 goto cannot_emulate;
3694 c->dst.val = (s32) c->src.val;
3696 case 0x6c: /* insb */
3697 case 0x6d: /* insw/insd */
3698 c->src.val = c->regs[VCPU_REGS_RDX];
3700 case 0x6e: /* outsb */
3701 case 0x6f: /* outsw/outsd */
3702 c->dst.val = c->regs[VCPU_REGS_RDX];
3705 case 0x70 ... 0x7f: /* jcc (short) */
3706 if (test_cc(c->b, ctxt->eflags))
3707 jmp_rel(c, c->src.val);
3709 case 0x80 ... 0x83: /* Grp1 */
3710 switch (c->modrm_reg) {
3731 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
3733 case 0x86 ... 0x87: /* xchg */
3735 /* Write back the register source. */
3736 c->src.val = c->dst.val;
3737 write_register_operand(&c->src);
3739 * Write back the memory destination with implicit LOCK
3742 c->dst.val = c->src.orig_val;
3745 case 0x8c: /* mov r/m, sreg */
3746 if (c->modrm_reg > VCPU_SREG_GS) {
3747 rc = emulate_ud(ctxt);
3750 c->dst.val = ops->get_segment_selector(c->modrm_reg, ctxt->vcpu);
3752 case 0x8d: /* lea r16/r32, m */
3753 c->dst.val = c->src.addr.mem.ea;
3755 case 0x8e: { /* mov seg, r/m16 */
3760 if (c->modrm_reg == VCPU_SREG_CS ||
3761 c->modrm_reg > VCPU_SREG_GS) {
3762 rc = emulate_ud(ctxt);
3766 if (c->modrm_reg == VCPU_SREG_SS)
3767 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3769 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
3771 c->dst.type = OP_NONE; /* Disable writeback. */
3774 case 0x8f: /* pop (sole member of Grp1a) */
3775 rc = emulate_grp1a(ctxt, ops);
3777 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3778 if (c->dst.addr.reg == &c->regs[VCPU_REGS_RAX])
3781 case 0x98: /* cbw/cwde/cdqe */
3782 switch (c->op_bytes) {
3783 case 2: c->dst.val = (s8)c->dst.val; break;
3784 case 4: c->dst.val = (s16)c->dst.val; break;
3785 case 8: c->dst.val = (s32)c->dst.val; break;
3788 case 0x9c: /* pushf */
3789 c->src.val = (unsigned long) ctxt->eflags;
3792 case 0x9d: /* popf */
3793 c->dst.type = OP_REG;
3794 c->dst.addr.reg = &ctxt->eflags;
3795 c->dst.bytes = c->op_bytes;
3796 rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
3798 case 0xa6 ... 0xa7: /* cmps */
3800 case 0xa8 ... 0xa9: /* test ax, imm */
3802 case 0xae ... 0xaf: /* scas */
3807 case 0xc3: /* ret */
3808 c->dst.type = OP_REG;
3809 c->dst.addr.reg = &c->eip;
3810 c->dst.bytes = c->op_bytes;
3811 goto pop_instruction;
3812 case 0xc4: /* les */
3813 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_ES);
3815 case 0xc5: /* lds */
3816 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_DS);
3818 case 0xcb: /* ret far */
3819 rc = emulate_ret_far(ctxt, ops);
3821 case 0xcc: /* int3 */
3824 case 0xcd: /* int n */
3827 rc = emulate_int(ctxt, ops, irq);
3829 case 0xce: /* into */
3830 if (ctxt->eflags & EFLG_OF) {
3835 case 0xcf: /* iret */
3836 rc = emulate_iret(ctxt, ops);
3838 case 0xd0 ... 0xd1: /* Grp2 */
3841 case 0xd2 ... 0xd3: /* Grp2 */
3842 c->src.val = c->regs[VCPU_REGS_RCX];
3845 case 0xe0 ... 0xe2: /* loop/loopz/loopnz */
3846 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3847 if (address_mask(c, c->regs[VCPU_REGS_RCX]) != 0 &&
3848 (c->b == 0xe2 || test_cc(c->b ^ 0x5, ctxt->eflags)))
3849 jmp_rel(c, c->src.val);
3851 case 0xe3: /* jcxz/jecxz/jrcxz */
3852 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0)
3853 jmp_rel(c, c->src.val);
3855 case 0xe4: /* inb */
3858 case 0xe6: /* outb */
3859 case 0xe7: /* out */
3861 case 0xe8: /* call (near) */ {
3862 long int rel = c->src.val;
3863 c->src.val = (unsigned long) c->eip;
3868 case 0xe9: /* jmp rel */
3870 case 0xea: { /* jmp far */
3873 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
3875 if (load_segment_descriptor(ctxt, ops, sel, VCPU_SREG_CS))
3879 memcpy(&c->eip, c->src.valptr, c->op_bytes);
3883 jmp: /* jmp rel short */
3884 jmp_rel(c, c->src.val);
3885 c->dst.type = OP_NONE; /* Disable writeback. */
3887 case 0xec: /* in al,dx */
3888 case 0xed: /* in (e/r)ax,dx */
3889 c->src.val = c->regs[VCPU_REGS_RDX];
3891 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
3893 goto done; /* IO is needed */
3895 case 0xee: /* out dx,al */
3896 case 0xef: /* out dx,(e/r)ax */
3897 c->dst.val = c->regs[VCPU_REGS_RDX];
3899 ops->pio_out_emulated(c->src.bytes, c->dst.val,
3900 &c->src.val, 1, ctxt->vcpu);
3901 c->dst.type = OP_NONE; /* Disable writeback. */
3903 case 0xf4: /* hlt */
3904 ctxt->vcpu->arch.halt_request = 1;
3906 case 0xf5: /* cmc */
3907 /* complement carry flag from eflags reg */
3908 ctxt->eflags ^= EFLG_CF;
3910 case 0xf6 ... 0xf7: /* Grp3 */
3911 rc = emulate_grp3(ctxt, ops);
3913 case 0xf8: /* clc */
3914 ctxt->eflags &= ~EFLG_CF;
3916 case 0xf9: /* stc */
3917 ctxt->eflags |= EFLG_CF;
3919 case 0xfa: /* cli */
3920 if (emulator_bad_iopl(ctxt, ops)) {
3921 rc = emulate_gp(ctxt, 0);
3924 ctxt->eflags &= ~X86_EFLAGS_IF;
3926 case 0xfb: /* sti */
3927 if (emulator_bad_iopl(ctxt, ops)) {
3928 rc = emulate_gp(ctxt, 0);
3931 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3932 ctxt->eflags |= X86_EFLAGS_IF;
3935 case 0xfc: /* cld */
3936 ctxt->eflags &= ~EFLG_DF;
3938 case 0xfd: /* std */
3939 ctxt->eflags |= EFLG_DF;
3941 case 0xfe: /* Grp4 */
3943 rc = emulate_grp45(ctxt);
3945 case 0xff: /* Grp5 */
3946 if (c->modrm_reg == 5)
3950 goto cannot_emulate;
3953 if (rc != X86EMUL_CONTINUE)
3957 rc = writeback(ctxt, ops);
3958 if (rc != X86EMUL_CONTINUE)
3962 * restore dst type in case the decoding will be reused
3963 * (happens for string instruction )
3965 c->dst.type = saved_dst_type;
3967 if ((c->d & SrcMask) == SrcSI)
3968 string_addr_inc(ctxt, seg_override(ctxt, ops, c),
3969 VCPU_REGS_RSI, &c->src);
3971 if ((c->d & DstMask) == DstDI)
3972 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
3975 if (c->rep_prefix && (c->d & String)) {
3976 struct read_cache *r = &ctxt->decode.io_read;
3977 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3979 if (!string_insn_completed(ctxt)) {
3981 * Re-enter guest when pio read ahead buffer is empty
3982 * or, if it is not used, after each 1024 iteration.
3984 if ((r->end != 0 || c->regs[VCPU_REGS_RCX] & 0x3ff) &&
3985 (r->end == 0 || r->end != r->pos)) {
3987 * Reset read cache. Usually happens before
3988 * decode, but since instruction is restarted
3989 * we have to do it here.
3991 ctxt->decode.mem_read.end = 0;
3992 return EMULATION_RESTART;
3994 goto done; /* skip rip writeback */
4001 if (rc == X86EMUL_PROPAGATE_FAULT)
4002 ctxt->have_exception = true;
4003 if (rc == X86EMUL_INTERCEPTED)
4004 return EMULATION_INTERCEPTED;
4006 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4010 case 0x01: /* lgdt, lidt, lmsw */
4011 switch (c->modrm_reg) {
4013 unsigned long address;
4015 case 0: /* vmcall */
4016 if (c->modrm_mod != 3 || c->modrm_rm != 1)
4017 goto cannot_emulate;
4019 rc = kvm_fix_hypercall(ctxt->vcpu);
4020 if (rc != X86EMUL_CONTINUE)
4023 /* Let the processor re-execute the fixed hypercall */
4025 /* Disable writeback. */
4026 c->dst.type = OP_NONE;
4029 rc = read_descriptor(ctxt, ops, c->src.addr.mem,
4030 &size, &address, c->op_bytes);
4031 if (rc != X86EMUL_CONTINUE)
4033 realmode_lgdt(ctxt->vcpu, size, address);
4034 /* Disable writeback. */
4035 c->dst.type = OP_NONE;
4037 case 3: /* lidt/vmmcall */
4038 if (c->modrm_mod == 3) {
4039 switch (c->modrm_rm) {
4041 rc = kvm_fix_hypercall(ctxt->vcpu);
4044 goto cannot_emulate;
4047 rc = read_descriptor(ctxt, ops, c->src.addr.mem,
4050 if (rc != X86EMUL_CONTINUE)
4052 realmode_lidt(ctxt->vcpu, size, address);
4054 /* Disable writeback. */
4055 c->dst.type = OP_NONE;
4059 c->dst.val = ops->get_cr(0, ctxt->vcpu);
4062 ops->set_cr(0, (ops->get_cr(0, ctxt->vcpu) & ~0x0eul) |
4063 (c->src.val & 0x0f), ctxt->vcpu);
4064 c->dst.type = OP_NONE;
4066 case 5: /* not defined */
4068 rc = X86EMUL_PROPAGATE_FAULT;
4071 rc = em_invlpg(ctxt);
4074 goto cannot_emulate;
4077 case 0x05: /* syscall */
4078 rc = emulate_syscall(ctxt, ops);
4081 emulate_clts(ctxt->vcpu);
4083 case 0x09: /* wbinvd */
4084 kvm_emulate_wbinvd(ctxt->vcpu);
4086 case 0x08: /* invd */
4087 case 0x0d: /* GrpP (prefetch) */
4088 case 0x18: /* Grp16 (prefetch/nop) */
4090 case 0x20: /* mov cr, reg */
4091 c->dst.val = ops->get_cr(c->modrm_reg, ctxt->vcpu);
4093 case 0x21: /* mov from dr to reg */
4094 ops->get_dr(c->modrm_reg, &c->dst.val, ctxt->vcpu);
4096 case 0x22: /* mov reg, cr */
4097 if (ops->set_cr(c->modrm_reg, c->src.val, ctxt->vcpu)) {
4098 emulate_gp(ctxt, 0);
4099 rc = X86EMUL_PROPAGATE_FAULT;
4102 c->dst.type = OP_NONE;
4104 case 0x23: /* mov from reg to dr */
4105 if (ops->set_dr(c->modrm_reg, c->src.val &
4106 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4107 ~0ULL : ~0U), ctxt->vcpu) < 0) {
4108 /* #UD condition is already handled by the code above */
4109 emulate_gp(ctxt, 0);
4110 rc = X86EMUL_PROPAGATE_FAULT;
4114 c->dst.type = OP_NONE; /* no writeback */
4118 msr_data = (u32)c->regs[VCPU_REGS_RAX]
4119 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
4120 if (ops->set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) {
4121 emulate_gp(ctxt, 0);
4122 rc = X86EMUL_PROPAGATE_FAULT;
4125 rc = X86EMUL_CONTINUE;
4129 if (ops->get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) {
4130 emulate_gp(ctxt, 0);
4131 rc = X86EMUL_PROPAGATE_FAULT;
4134 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
4135 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
4137 rc = X86EMUL_CONTINUE;
4139 case 0x34: /* sysenter */
4140 rc = emulate_sysenter(ctxt, ops);
4142 case 0x35: /* sysexit */
4143 rc = emulate_sysexit(ctxt, ops);
4145 case 0x40 ... 0x4f: /* cmov */
4146 c->dst.val = c->dst.orig_val = c->src.val;
4147 if (!test_cc(c->b, ctxt->eflags))
4148 c->dst.type = OP_NONE; /* no writeback */
4150 case 0x80 ... 0x8f: /* jnz rel, etc*/
4151 if (test_cc(c->b, ctxt->eflags))
4152 jmp_rel(c, c->src.val);
4154 case 0x90 ... 0x9f: /* setcc r/m8 */
4155 c->dst.val = test_cc(c->b, ctxt->eflags);
4157 case 0xa0: /* push fs */
4158 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
4160 case 0xa1: /* pop fs */
4161 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
4165 c->dst.type = OP_NONE;
4166 /* only subword offset */
4167 c->src.val &= (c->dst.bytes << 3) - 1;
4168 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
4170 case 0xa4: /* shld imm8, r, r/m */
4171 case 0xa5: /* shld cl, r, r/m */
4172 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
4174 case 0xa8: /* push gs */
4175 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
4177 case 0xa9: /* pop gs */
4178 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
4182 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
4184 case 0xac: /* shrd imm8, r, r/m */
4185 case 0xad: /* shrd cl, r, r/m */
4186 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
4188 case 0xae: /* clflush */
4190 case 0xb0 ... 0xb1: /* cmpxchg */
4192 * Save real source value, then compare EAX against
4195 c->src.orig_val = c->src.val;
4196 c->src.val = c->regs[VCPU_REGS_RAX];
4197 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
4198 if (ctxt->eflags & EFLG_ZF) {
4199 /* Success: write back to memory. */
4200 c->dst.val = c->src.orig_val;
4202 /* Failure: write the value we saw to EAX. */
4203 c->dst.type = OP_REG;
4204 c->dst.addr.reg = (unsigned long *)&c->regs[VCPU_REGS_RAX];
4207 case 0xb2: /* lss */
4208 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_SS);
4212 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
4214 case 0xb4: /* lfs */
4215 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_FS);
4217 case 0xb5: /* lgs */
4218 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_GS);
4220 case 0xb6 ... 0xb7: /* movzx */
4221 c->dst.bytes = c->op_bytes;
4222 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
4225 case 0xba: /* Grp8 */
4226 switch (c->modrm_reg & 3) {
4239 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
4241 case 0xbc: { /* bsf */
4243 __asm__ ("bsf %2, %0; setz %1"
4244 : "=r"(c->dst.val), "=q"(zf)
4246 ctxt->eflags &= ~X86_EFLAGS_ZF;
4248 ctxt->eflags |= X86_EFLAGS_ZF;
4249 c->dst.type = OP_NONE; /* Disable writeback. */
4253 case 0xbd: { /* bsr */
4255 __asm__ ("bsr %2, %0; setz %1"
4256 : "=r"(c->dst.val), "=q"(zf)
4258 ctxt->eflags &= ~X86_EFLAGS_ZF;
4260 ctxt->eflags |= X86_EFLAGS_ZF;
4261 c->dst.type = OP_NONE; /* Disable writeback. */
4265 case 0xbe ... 0xbf: /* movsx */
4266 c->dst.bytes = c->op_bytes;
4267 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
4270 case 0xc0 ... 0xc1: /* xadd */
4271 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
4272 /* Write back the register source. */
4273 c->src.val = c->dst.orig_val;
4274 write_register_operand(&c->src);
4276 case 0xc3: /* movnti */
4277 c->dst.bytes = c->op_bytes;
4278 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
4281 case 0xc7: /* Grp9 (cmpxchg8b) */
4282 rc = emulate_grp9(ctxt, ops);
4285 goto cannot_emulate;
4288 if (rc != X86EMUL_CONTINUE)
4294 return EMULATION_FAILED;