1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
32 * Opcode effective-address decode tables.
33 * Note that we only emulate instructions that have at least one memory
34 * operand (excluding implicit stack references). We assume that stack
35 * references and instruction fetches will never occur in special memory
36 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
40 /* Operand sizes: 8-bit operands or specified/overridden size. */
41 #define ByteOp (1<<0) /* 8-bit operands. */
42 /* Destination operand type. */
43 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
44 #define DstReg (2<<1) /* Register operand. */
45 #define DstMem (3<<1) /* Memory operand. */
46 #define DstAcc (4<<1) /* Destination Accumulator */
47 #define DstDI (5<<1) /* Destination is in ES:(E)DI */
48 #define DstMem64 (6<<1) /* 64bit memory operand */
49 #define DstImmUByte (7<<1) /* 8-bit unsigned immediate operand */
50 #define DstMask (7<<1)
51 /* Source operand type. */
52 #define SrcNone (0<<4) /* No source operand. */
53 #define SrcReg (1<<4) /* Register operand. */
54 #define SrcMem (2<<4) /* Memory operand. */
55 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
56 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
57 #define SrcImm (5<<4) /* Immediate operand. */
58 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
59 #define SrcOne (7<<4) /* Implied '1' */
60 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
61 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
62 #define SrcSI (0xa<<4) /* Source is in the DS:RSI */
63 #define SrcImmFAddr (0xb<<4) /* Source is immediate far address */
64 #define SrcMemFAddr (0xc<<4) /* Source is far address in memory */
65 #define SrcAcc (0xd<<4) /* Source Accumulator */
66 #define SrcImmU16 (0xe<<4) /* Immediate operand, unsigned, 16 bits */
67 #define SrcMask (0xf<<4)
68 /* Generic ModRM decode. */
70 /* Destination is only written; never read. */
73 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
74 #define String (1<<12) /* String instruction (rep capable) */
75 #define Stack (1<<13) /* Stack instruction (push/pop) */
76 #define GroupMask (7<<14) /* Opcode uses one of the group mechanisms */
77 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
78 #define GroupDual (2<<14) /* Alternate decoding of mod == 3 */
79 #define Prefix (3<<14) /* Instruction varies with 66/f2/f3 prefix */
80 #define RMExt (4<<14) /* Opcode extension in ModRM r/m if mod == 3 */
81 #define Sse (1<<17) /* SSE Vector instruction */
83 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
84 #define VendorSpecific (1<<22) /* Vendor specific instruction */
85 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
86 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
87 #define Undefined (1<<25) /* No Such Instruction */
88 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
89 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
91 /* Source 2 operand type */
92 #define Src2None (0<<29)
93 #define Src2CL (1<<29)
94 #define Src2ImmByte (2<<29)
95 #define Src2One (3<<29)
96 #define Src2Imm (4<<29)
97 #define Src2Mask (7<<29)
100 #define X3(x...) X2(x), x
101 #define X4(x...) X2(x), X2(x)
102 #define X5(x...) X4(x), x
103 #define X6(x...) X4(x), X2(x)
104 #define X7(x...) X4(x), X3(x)
105 #define X8(x...) X4(x), X4(x)
106 #define X16(x...) X8(x), X8(x)
112 int (*execute)(struct x86_emulate_ctxt *ctxt);
113 struct opcode *group;
114 struct group_dual *gdual;
115 struct gprefix *gprefix;
117 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
121 struct opcode mod012[8];
122 struct opcode mod3[8];
126 struct opcode pfx_no;
127 struct opcode pfx_66;
128 struct opcode pfx_f2;
129 struct opcode pfx_f3;
132 /* EFLAGS bit definitions. */
133 #define EFLG_ID (1<<21)
134 #define EFLG_VIP (1<<20)
135 #define EFLG_VIF (1<<19)
136 #define EFLG_AC (1<<18)
137 #define EFLG_VM (1<<17)
138 #define EFLG_RF (1<<16)
139 #define EFLG_IOPL (3<<12)
140 #define EFLG_NT (1<<14)
141 #define EFLG_OF (1<<11)
142 #define EFLG_DF (1<<10)
143 #define EFLG_IF (1<<9)
144 #define EFLG_TF (1<<8)
145 #define EFLG_SF (1<<7)
146 #define EFLG_ZF (1<<6)
147 #define EFLG_AF (1<<4)
148 #define EFLG_PF (1<<2)
149 #define EFLG_CF (1<<0)
151 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
152 #define EFLG_RESERVED_ONE_MASK 2
155 * Instruction emulation:
156 * Most instructions are emulated directly via a fragment of inline assembly
157 * code. This allows us to save/restore EFLAGS and thus very easily pick up
158 * any modified flags.
161 #if defined(CONFIG_X86_64)
162 #define _LO32 "k" /* force 32-bit operand */
163 #define _STK "%%rsp" /* stack pointer */
164 #elif defined(__i386__)
165 #define _LO32 "" /* force 32-bit operand */
166 #define _STK "%%esp" /* stack pointer */
170 * These EFLAGS bits are restored from saved value during emulation, and
171 * any changes are written back to the saved value after emulation.
173 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
175 /* Before executing instruction: restore necessary bits in EFLAGS. */
176 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
177 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
178 "movl %"_sav",%"_LO32 _tmp"; " \
181 "movl %"_msk",%"_LO32 _tmp"; " \
182 "andl %"_LO32 _tmp",("_STK"); " \
184 "notl %"_LO32 _tmp"; " \
185 "andl %"_LO32 _tmp",("_STK"); " \
186 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
188 "orl %"_LO32 _tmp",("_STK"); " \
192 /* After executing instruction: write-back necessary bits in EFLAGS. */
193 #define _POST_EFLAGS(_sav, _msk, _tmp) \
194 /* _sav |= EFLAGS & _msk; */ \
197 "andl %"_msk",%"_LO32 _tmp"; " \
198 "orl %"_LO32 _tmp",%"_sav"; "
206 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix, _dsttype) \
208 __asm__ __volatile__ ( \
209 _PRE_EFLAGS("0", "4", "2") \
210 _op _suffix " %"_x"3,%1; " \
211 _POST_EFLAGS("0", "4", "2") \
212 : "=m" (_eflags), "+q" (*(_dsttype*)&(_dst).val),\
214 : _y ((_src).val), "i" (EFLAGS_MASK)); \
218 /* Raw emulation: instruction has two explicit operands. */
219 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
221 unsigned long _tmp; \
223 switch ((_dst).bytes) { \
225 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w",u16);\
228 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l",u32);\
231 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q",u64)); \
236 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
238 unsigned long _tmp; \
239 switch ((_dst).bytes) { \
241 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b",u8); \
244 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
245 _wx, _wy, _lx, _ly, _qx, _qy); \
250 /* Source operand is byte-sized and may be restricted to just %cl. */
251 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
252 __emulate_2op(_op, _src, _dst, _eflags, \
253 "b", "c", "b", "c", "b", "c", "b", "c")
255 /* Source operand is byte, word, long or quad sized. */
256 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
257 __emulate_2op(_op, _src, _dst, _eflags, \
258 "b", "q", "w", "r", _LO32, "r", "", "r")
260 /* Source operand is word, long or quad sized. */
261 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
262 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
263 "w", "r", _LO32, "r", "", "r")
265 /* Instruction has three operands and one operand is stored in ECX register */
266 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
268 unsigned long _tmp; \
269 _type _clv = (_cl).val; \
270 _type _srcv = (_src).val; \
271 _type _dstv = (_dst).val; \
273 __asm__ __volatile__ ( \
274 _PRE_EFLAGS("0", "5", "2") \
275 _op _suffix " %4,%1 \n" \
276 _POST_EFLAGS("0", "5", "2") \
277 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
278 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
281 (_cl).val = (unsigned long) _clv; \
282 (_src).val = (unsigned long) _srcv; \
283 (_dst).val = (unsigned long) _dstv; \
286 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
288 switch ((_dst).bytes) { \
290 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
291 "w", unsigned short); \
294 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
295 "l", unsigned int); \
298 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
299 "q", unsigned long)); \
304 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
306 unsigned long _tmp; \
308 __asm__ __volatile__ ( \
309 _PRE_EFLAGS("0", "3", "2") \
310 _op _suffix " %1; " \
311 _POST_EFLAGS("0", "3", "2") \
312 : "=m" (_eflags), "+m" ((_dst).val), \
314 : "i" (EFLAGS_MASK)); \
317 /* Instruction has only one explicit operand (no source operand). */
318 #define emulate_1op(_op, _dst, _eflags) \
320 switch ((_dst).bytes) { \
321 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
322 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
323 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
324 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
328 #define __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, _suffix) \
330 unsigned long _tmp; \
332 __asm__ __volatile__ ( \
333 _PRE_EFLAGS("0", "4", "1") \
334 _op _suffix " %5; " \
335 _POST_EFLAGS("0", "4", "1") \
336 : "=m" (_eflags), "=&r" (_tmp), \
337 "+a" (_rax), "+d" (_rdx) \
338 : "i" (EFLAGS_MASK), "m" ((_src).val), \
339 "a" (_rax), "d" (_rdx)); \
342 #define __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _suffix, _ex) \
344 unsigned long _tmp; \
346 __asm__ __volatile__ ( \
347 _PRE_EFLAGS("0", "5", "1") \
349 _op _suffix " %6; " \
351 _POST_EFLAGS("0", "5", "1") \
352 ".pushsection .fixup,\"ax\" \n\t" \
353 "3: movb $1, %4 \n\t" \
356 _ASM_EXTABLE(1b, 3b) \
357 : "=m" (_eflags), "=&r" (_tmp), \
358 "+a" (_rax), "+d" (_rdx), "+qm"(_ex) \
359 : "i" (EFLAGS_MASK), "m" ((_src).val), \
360 "a" (_rax), "d" (_rdx)); \
363 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
364 #define emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags) \
366 switch((_src).bytes) { \
368 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
372 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
376 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
380 ON64(__emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
386 #define emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _ex) \
388 switch((_src).bytes) { \
390 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
391 _eflags, "b", _ex); \
394 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
395 _eflags, "w", _ex); \
398 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
399 _eflags, "l", _ex); \
402 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
403 _eflags, "q", _ex)); \
408 /* Fetch next part of the instruction being emulated. */
409 #define insn_fetch(_type, _size, _eip) \
410 ({ unsigned long _x; \
411 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
412 if (rc != X86EMUL_CONTINUE) \
418 #define insn_fetch_arr(_arr, _size, _eip) \
419 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
420 if (rc != X86EMUL_CONTINUE) \
425 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
426 enum x86_intercept intercept,
427 enum x86_intercept_stage stage)
429 struct x86_instruction_info info = {
430 .intercept = intercept,
431 .rep_prefix = ctxt->decode.rep_prefix,
432 .modrm_mod = ctxt->decode.modrm_mod,
433 .modrm_reg = ctxt->decode.modrm_reg,
434 .modrm_rm = ctxt->decode.modrm_rm,
435 .src_val = ctxt->decode.src.val64,
436 .src_bytes = ctxt->decode.src.bytes,
437 .dst_bytes = ctxt->decode.dst.bytes,
438 .ad_bytes = ctxt->decode.ad_bytes,
439 .next_rip = ctxt->eip,
442 return ctxt->ops->intercept(ctxt, &info, stage);
445 static inline unsigned long ad_mask(struct decode_cache *c)
447 return (1UL << (c->ad_bytes << 3)) - 1;
450 /* Access/update address held in a register, based on addressing mode. */
451 static inline unsigned long
452 address_mask(struct decode_cache *c, unsigned long reg)
454 if (c->ad_bytes == sizeof(unsigned long))
457 return reg & ad_mask(c);
460 static inline unsigned long
461 register_address(struct decode_cache *c, unsigned long reg)
463 return address_mask(c, reg);
467 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
469 if (c->ad_bytes == sizeof(unsigned long))
472 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
475 static inline void jmp_rel(struct decode_cache *c, int rel)
477 register_address_increment(c, &c->eip, rel);
480 static u32 desc_limit_scaled(struct desc_struct *desc)
482 u32 limit = get_desc_limit(desc);
484 return desc->g ? (limit << 12) | 0xfff : limit;
487 static void set_seg_override(struct decode_cache *c, int seg)
489 c->has_seg_override = true;
490 c->seg_override = seg;
493 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
494 struct x86_emulate_ops *ops, int seg)
496 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
499 return ops->get_cached_segment_base(ctxt, seg);
502 static unsigned seg_override(struct x86_emulate_ctxt *ctxt,
503 struct x86_emulate_ops *ops,
504 struct decode_cache *c)
506 if (!c->has_seg_override)
509 return c->seg_override;
512 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
513 u32 error, bool valid)
515 ctxt->exception.vector = vec;
516 ctxt->exception.error_code = error;
517 ctxt->exception.error_code_valid = valid;
518 return X86EMUL_PROPAGATE_FAULT;
521 static int emulate_db(struct x86_emulate_ctxt *ctxt)
523 return emulate_exception(ctxt, DB_VECTOR, 0, false);
526 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
528 return emulate_exception(ctxt, GP_VECTOR, err, true);
531 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
533 return emulate_exception(ctxt, SS_VECTOR, err, true);
536 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
538 return emulate_exception(ctxt, UD_VECTOR, 0, false);
541 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
543 return emulate_exception(ctxt, TS_VECTOR, err, true);
546 static int emulate_de(struct x86_emulate_ctxt *ctxt)
548 return emulate_exception(ctxt, DE_VECTOR, 0, false);
551 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
553 return emulate_exception(ctxt, NM_VECTOR, 0, false);
556 static int __linearize(struct x86_emulate_ctxt *ctxt,
557 struct segmented_address addr,
558 unsigned size, bool write, bool fetch,
561 struct decode_cache *c = &ctxt->decode;
562 struct desc_struct desc;
568 la = seg_base(ctxt, ctxt->ops, addr.seg) + addr.ea;
569 switch (ctxt->mode) {
570 case X86EMUL_MODE_REAL:
572 case X86EMUL_MODE_PROT64:
573 if (((signed long)la << 16) >> 16 != la)
574 return emulate_gp(ctxt, 0);
577 usable = ctxt->ops->get_cached_descriptor(ctxt, &desc, NULL,
581 /* code segment or read-only data segment */
582 if (((desc.type & 8) || !(desc.type & 2)) && write)
584 /* unreadable code segment */
585 if (!fetch && (desc.type & 8) && !(desc.type & 2))
587 lim = desc_limit_scaled(&desc);
588 if ((desc.type & 8) || !(desc.type & 4)) {
589 /* expand-up segment */
590 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
593 /* exapand-down segment */
594 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
596 lim = desc.d ? 0xffffffff : 0xffff;
597 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
600 cpl = ctxt->ops->cpl(ctxt);
601 rpl = ctxt->ops->get_segment_selector(ctxt, addr.seg) & 3;
603 if (!(desc.type & 8)) {
607 } else if ((desc.type & 8) && !(desc.type & 4)) {
608 /* nonconforming code segment */
611 } else if ((desc.type & 8) && (desc.type & 4)) {
612 /* conforming code segment */
618 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : c->ad_bytes != 8)
621 return X86EMUL_CONTINUE;
623 if (addr.seg == VCPU_SREG_SS)
624 return emulate_ss(ctxt, addr.seg);
626 return emulate_gp(ctxt, addr.seg);
629 static int linearize(struct x86_emulate_ctxt *ctxt,
630 struct segmented_address addr,
631 unsigned size, bool write,
634 return __linearize(ctxt, addr, size, write, false, linear);
638 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
639 struct segmented_address addr,
646 rc = linearize(ctxt, addr, size, false, &linear);
647 if (rc != X86EMUL_CONTINUE)
649 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
652 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
653 struct x86_emulate_ops *ops,
654 unsigned long eip, u8 *dest)
656 struct fetch_cache *fc = &ctxt->decode.fetch;
660 if (eip == fc->end) {
661 unsigned long linear;
662 struct segmented_address addr = { .seg=VCPU_SREG_CS, .ea=eip};
663 cur_size = fc->end - fc->start;
664 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
665 rc = __linearize(ctxt, addr, size, false, true, &linear);
666 if (rc != X86EMUL_CONTINUE)
668 rc = ops->fetch(ctxt, linear, fc->data + cur_size,
669 size, &ctxt->exception);
670 if (rc != X86EMUL_CONTINUE)
674 *dest = fc->data[eip - fc->start];
675 return X86EMUL_CONTINUE;
678 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
679 struct x86_emulate_ops *ops,
680 unsigned long eip, void *dest, unsigned size)
684 /* x86 instructions are limited to 15 bytes. */
685 if (eip + size - ctxt->eip > 15)
686 return X86EMUL_UNHANDLEABLE;
688 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
689 if (rc != X86EMUL_CONTINUE)
692 return X86EMUL_CONTINUE;
696 * Given the 'reg' portion of a ModRM byte, and a register block, return a
697 * pointer into the block that addresses the relevant register.
698 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
700 static void *decode_register(u8 modrm_reg, unsigned long *regs,
705 p = ®s[modrm_reg];
706 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
707 p = (unsigned char *)®s[modrm_reg & 3] + 1;
711 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
712 struct x86_emulate_ops *ops,
713 struct segmented_address addr,
714 u16 *size, unsigned long *address, int op_bytes)
721 rc = segmented_read_std(ctxt, addr, size, 2);
722 if (rc != X86EMUL_CONTINUE)
725 rc = segmented_read_std(ctxt, addr, address, op_bytes);
729 static int test_cc(unsigned int condition, unsigned int flags)
733 switch ((condition & 15) >> 1) {
735 rc |= (flags & EFLG_OF);
737 case 1: /* b/c/nae */
738 rc |= (flags & EFLG_CF);
741 rc |= (flags & EFLG_ZF);
744 rc |= (flags & (EFLG_CF|EFLG_ZF));
747 rc |= (flags & EFLG_SF);
750 rc |= (flags & EFLG_PF);
753 rc |= (flags & EFLG_ZF);
756 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
760 /* Odd condition identifiers (lsb == 1) have inverted sense. */
761 return (!!rc ^ (condition & 1));
764 static void fetch_register_operand(struct operand *op)
768 op->val = *(u8 *)op->addr.reg;
771 op->val = *(u16 *)op->addr.reg;
774 op->val = *(u32 *)op->addr.reg;
777 op->val = *(u64 *)op->addr.reg;
782 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
784 ctxt->ops->get_fpu(ctxt);
786 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
787 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
788 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
789 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
790 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
791 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
792 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
793 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
795 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
796 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
797 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
798 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
799 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
800 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
801 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
802 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
806 ctxt->ops->put_fpu(ctxt);
809 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
812 ctxt->ops->get_fpu(ctxt);
814 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
815 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
816 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
817 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
818 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
819 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
820 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
821 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
823 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
824 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
825 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
826 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
827 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
828 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
829 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
830 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
834 ctxt->ops->put_fpu(ctxt);
837 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
839 struct decode_cache *c,
842 unsigned reg = c->modrm_reg;
843 int highbyte_regs = c->rex_prefix == 0;
846 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
852 read_sse_reg(ctxt, &op->vec_val, reg);
857 if ((c->d & ByteOp) && !inhibit_bytereg) {
858 op->addr.reg = decode_register(reg, c->regs, highbyte_regs);
861 op->addr.reg = decode_register(reg, c->regs, 0);
862 op->bytes = c->op_bytes;
864 fetch_register_operand(op);
865 op->orig_val = op->val;
868 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
869 struct x86_emulate_ops *ops,
872 struct decode_cache *c = &ctxt->decode;
874 int index_reg = 0, base_reg = 0, scale;
875 int rc = X86EMUL_CONTINUE;
879 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
880 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
881 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
884 c->modrm = insn_fetch(u8, 1, c->eip);
885 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
886 c->modrm_reg |= (c->modrm & 0x38) >> 3;
887 c->modrm_rm |= (c->modrm & 0x07);
888 c->modrm_seg = VCPU_SREG_DS;
890 if (c->modrm_mod == 3) {
892 op->bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
893 op->addr.reg = decode_register(c->modrm_rm,
894 c->regs, c->d & ByteOp);
898 op->addr.xmm = c->modrm_rm;
899 read_sse_reg(ctxt, &op->vec_val, c->modrm_rm);
902 fetch_register_operand(op);
908 if (c->ad_bytes == 2) {
909 unsigned bx = c->regs[VCPU_REGS_RBX];
910 unsigned bp = c->regs[VCPU_REGS_RBP];
911 unsigned si = c->regs[VCPU_REGS_RSI];
912 unsigned di = c->regs[VCPU_REGS_RDI];
914 /* 16-bit ModR/M decode. */
915 switch (c->modrm_mod) {
917 if (c->modrm_rm == 6)
918 modrm_ea += insn_fetch(u16, 2, c->eip);
921 modrm_ea += insn_fetch(s8, 1, c->eip);
924 modrm_ea += insn_fetch(u16, 2, c->eip);
927 switch (c->modrm_rm) {
947 if (c->modrm_mod != 0)
954 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
955 (c->modrm_rm == 6 && c->modrm_mod != 0))
956 c->modrm_seg = VCPU_SREG_SS;
957 modrm_ea = (u16)modrm_ea;
959 /* 32/64-bit ModR/M decode. */
960 if ((c->modrm_rm & 7) == 4) {
961 sib = insn_fetch(u8, 1, c->eip);
962 index_reg |= (sib >> 3) & 7;
966 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
967 modrm_ea += insn_fetch(s32, 4, c->eip);
969 modrm_ea += c->regs[base_reg];
971 modrm_ea += c->regs[index_reg] << scale;
972 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
973 if (ctxt->mode == X86EMUL_MODE_PROT64)
976 modrm_ea += c->regs[c->modrm_rm];
977 switch (c->modrm_mod) {
979 if (c->modrm_rm == 5)
980 modrm_ea += insn_fetch(s32, 4, c->eip);
983 modrm_ea += insn_fetch(s8, 1, c->eip);
986 modrm_ea += insn_fetch(s32, 4, c->eip);
990 op->addr.mem.ea = modrm_ea;
995 static int decode_abs(struct x86_emulate_ctxt *ctxt,
996 struct x86_emulate_ops *ops,
999 struct decode_cache *c = &ctxt->decode;
1000 int rc = X86EMUL_CONTINUE;
1003 switch (c->ad_bytes) {
1005 op->addr.mem.ea = insn_fetch(u16, 2, c->eip);
1008 op->addr.mem.ea = insn_fetch(u32, 4, c->eip);
1011 op->addr.mem.ea = insn_fetch(u64, 8, c->eip);
1018 static void fetch_bit_operand(struct decode_cache *c)
1022 if (c->dst.type == OP_MEM && c->src.type == OP_REG) {
1023 mask = ~(c->dst.bytes * 8 - 1);
1025 if (c->src.bytes == 2)
1026 sv = (s16)c->src.val & (s16)mask;
1027 else if (c->src.bytes == 4)
1028 sv = (s32)c->src.val & (s32)mask;
1030 c->dst.addr.mem.ea += (sv >> 3);
1033 /* only subword offset */
1034 c->src.val &= (c->dst.bytes << 3) - 1;
1037 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1038 struct x86_emulate_ops *ops,
1039 unsigned long addr, void *dest, unsigned size)
1042 struct read_cache *mc = &ctxt->decode.mem_read;
1045 int n = min(size, 8u);
1047 if (mc->pos < mc->end)
1050 rc = ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1052 if (rc != X86EMUL_CONTINUE)
1057 memcpy(dest, mc->data + mc->pos, n);
1062 return X86EMUL_CONTINUE;
1065 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1066 struct segmented_address addr,
1073 rc = linearize(ctxt, addr, size, false, &linear);
1074 if (rc != X86EMUL_CONTINUE)
1076 return read_emulated(ctxt, ctxt->ops, linear, data, size);
1079 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1080 struct segmented_address addr,
1087 rc = linearize(ctxt, addr, size, true, &linear);
1088 if (rc != X86EMUL_CONTINUE)
1090 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1094 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1095 struct segmented_address addr,
1096 const void *orig_data, const void *data,
1102 rc = linearize(ctxt, addr, size, true, &linear);
1103 if (rc != X86EMUL_CONTINUE)
1105 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1106 size, &ctxt->exception);
1109 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1110 struct x86_emulate_ops *ops,
1111 unsigned int size, unsigned short port,
1114 struct read_cache *rc = &ctxt->decode.io_read;
1116 if (rc->pos == rc->end) { /* refill pio read ahead */
1117 struct decode_cache *c = &ctxt->decode;
1118 unsigned int in_page, n;
1119 unsigned int count = c->rep_prefix ?
1120 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
1121 in_page = (ctxt->eflags & EFLG_DF) ?
1122 offset_in_page(c->regs[VCPU_REGS_RDI]) :
1123 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
1124 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1128 rc->pos = rc->end = 0;
1129 if (!ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1134 memcpy(dest, rc->data + rc->pos, size);
1139 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1140 struct x86_emulate_ops *ops,
1141 u16 selector, struct desc_ptr *dt)
1143 if (selector & 1 << 2) {
1144 struct desc_struct desc;
1145 memset (dt, 0, sizeof *dt);
1146 if (!ops->get_cached_descriptor(ctxt, &desc, NULL,
1150 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1151 dt->address = get_desc_base(&desc);
1153 ops->get_gdt(ctxt, dt);
1156 /* allowed just for 8 bytes segments */
1157 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1158 struct x86_emulate_ops *ops,
1159 u16 selector, struct desc_struct *desc)
1162 u16 index = selector >> 3;
1166 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1168 if (dt.size < index * 8 + 7)
1169 return emulate_gp(ctxt, selector & 0xfffc);
1170 addr = dt.address + index * 8;
1171 ret = ops->read_std(ctxt, addr, desc, sizeof *desc, &ctxt->exception);
1176 /* allowed just for 8 bytes segments */
1177 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1178 struct x86_emulate_ops *ops,
1179 u16 selector, struct desc_struct *desc)
1182 u16 index = selector >> 3;
1186 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1188 if (dt.size < index * 8 + 7)
1189 return emulate_gp(ctxt, selector & 0xfffc);
1191 addr = dt.address + index * 8;
1192 ret = ops->write_std(ctxt, addr, desc, sizeof *desc, &ctxt->exception);
1197 /* Does not support long mode */
1198 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1199 struct x86_emulate_ops *ops,
1200 u16 selector, int seg)
1202 struct desc_struct seg_desc;
1204 unsigned err_vec = GP_VECTOR;
1206 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1209 memset(&seg_desc, 0, sizeof seg_desc);
1211 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1212 || ctxt->mode == X86EMUL_MODE_REAL) {
1213 /* set real mode segment descriptor */
1214 set_desc_base(&seg_desc, selector << 4);
1215 set_desc_limit(&seg_desc, 0xffff);
1222 /* NULL selector is not valid for TR, CS and SS */
1223 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1227 /* TR should be in GDT only */
1228 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1231 if (null_selector) /* for NULL selector skip all following checks */
1234 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1235 if (ret != X86EMUL_CONTINUE)
1238 err_code = selector & 0xfffc;
1239 err_vec = GP_VECTOR;
1241 /* can't load system descriptor into segment selecor */
1242 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1246 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1252 cpl = ops->cpl(ctxt);
1257 * segment is not a writable data segment or segment
1258 * selector's RPL != CPL or segment selector's RPL != CPL
1260 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1264 if (!(seg_desc.type & 8))
1267 if (seg_desc.type & 4) {
1273 if (rpl > cpl || dpl != cpl)
1276 /* CS(RPL) <- CPL */
1277 selector = (selector & 0xfffc) | cpl;
1280 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1283 case VCPU_SREG_LDTR:
1284 if (seg_desc.s || seg_desc.type != 2)
1287 default: /* DS, ES, FS, or GS */
1289 * segment is not a data or readable code segment or
1290 * ((segment is a data or nonconforming code segment)
1291 * and (both RPL and CPL > DPL))
1293 if ((seg_desc.type & 0xa) == 0x8 ||
1294 (((seg_desc.type & 0xc) != 0xc) &&
1295 (rpl > dpl && cpl > dpl)))
1301 /* mark segment as accessed */
1303 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1304 if (ret != X86EMUL_CONTINUE)
1308 ops->set_segment_selector(ctxt, selector, seg);
1309 ops->set_cached_descriptor(ctxt, &seg_desc, 0, seg);
1310 return X86EMUL_CONTINUE;
1312 emulate_exception(ctxt, err_vec, err_code, true);
1313 return X86EMUL_PROPAGATE_FAULT;
1316 static void write_register_operand(struct operand *op)
1318 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1319 switch (op->bytes) {
1321 *(u8 *)op->addr.reg = (u8)op->val;
1324 *(u16 *)op->addr.reg = (u16)op->val;
1327 *op->addr.reg = (u32)op->val;
1328 break; /* 64b: zero-extend */
1330 *op->addr.reg = op->val;
1335 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1336 struct x86_emulate_ops *ops)
1339 struct decode_cache *c = &ctxt->decode;
1341 switch (c->dst.type) {
1343 write_register_operand(&c->dst);
1347 rc = segmented_cmpxchg(ctxt,
1353 rc = segmented_write(ctxt,
1357 if (rc != X86EMUL_CONTINUE)
1361 write_sse_reg(ctxt, &c->dst.vec_val, c->dst.addr.xmm);
1369 return X86EMUL_CONTINUE;
1372 static int em_push(struct x86_emulate_ctxt *ctxt)
1374 struct decode_cache *c = &ctxt->decode;
1375 struct segmented_address addr;
1377 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1378 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1379 addr.seg = VCPU_SREG_SS;
1381 /* Disable writeback. */
1382 c->dst.type = OP_NONE;
1383 return segmented_write(ctxt, addr, &c->src.val, c->op_bytes);
1386 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1387 struct x86_emulate_ops *ops,
1388 void *dest, int len)
1390 struct decode_cache *c = &ctxt->decode;
1392 struct segmented_address addr;
1394 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1395 addr.seg = VCPU_SREG_SS;
1396 rc = segmented_read(ctxt, addr, dest, len);
1397 if (rc != X86EMUL_CONTINUE)
1400 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1404 static int em_pop(struct x86_emulate_ctxt *ctxt)
1406 struct decode_cache *c = &ctxt->decode;
1408 return emulate_pop(ctxt, ctxt->ops, &c->dst.val, c->op_bytes);
1411 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1412 struct x86_emulate_ops *ops,
1413 void *dest, int len)
1416 unsigned long val, change_mask;
1417 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1418 int cpl = ops->cpl(ctxt);
1420 rc = emulate_pop(ctxt, ops, &val, len);
1421 if (rc != X86EMUL_CONTINUE)
1424 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1425 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1427 switch(ctxt->mode) {
1428 case X86EMUL_MODE_PROT64:
1429 case X86EMUL_MODE_PROT32:
1430 case X86EMUL_MODE_PROT16:
1432 change_mask |= EFLG_IOPL;
1434 change_mask |= EFLG_IF;
1436 case X86EMUL_MODE_VM86:
1438 return emulate_gp(ctxt, 0);
1439 change_mask |= EFLG_IF;
1441 default: /* real mode */
1442 change_mask |= (EFLG_IOPL | EFLG_IF);
1446 *(unsigned long *)dest =
1447 (ctxt->eflags & ~change_mask) | (val & change_mask);
1452 static int em_popf(struct x86_emulate_ctxt *ctxt)
1454 struct decode_cache *c = &ctxt->decode;
1456 c->dst.type = OP_REG;
1457 c->dst.addr.reg = &ctxt->eflags;
1458 c->dst.bytes = c->op_bytes;
1459 return emulate_popf(ctxt, ctxt->ops, &c->dst.val, c->op_bytes);
1462 static int emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1463 struct x86_emulate_ops *ops, int seg)
1465 struct decode_cache *c = &ctxt->decode;
1467 c->src.val = ops->get_segment_selector(ctxt, seg);
1469 return em_push(ctxt);
1472 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1473 struct x86_emulate_ops *ops, int seg)
1475 struct decode_cache *c = &ctxt->decode;
1476 unsigned long selector;
1479 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1480 if (rc != X86EMUL_CONTINUE)
1483 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1487 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1489 struct decode_cache *c = &ctxt->decode;
1490 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1491 int rc = X86EMUL_CONTINUE;
1492 int reg = VCPU_REGS_RAX;
1494 while (reg <= VCPU_REGS_RDI) {
1495 (reg == VCPU_REGS_RSP) ?
1496 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1499 if (rc != X86EMUL_CONTINUE)
1508 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1510 struct decode_cache *c = &ctxt->decode;
1512 c->src.val = (unsigned long)ctxt->eflags;
1513 return em_push(ctxt);
1516 static int em_popa(struct x86_emulate_ctxt *ctxt)
1518 struct decode_cache *c = &ctxt->decode;
1519 int rc = X86EMUL_CONTINUE;
1520 int reg = VCPU_REGS_RDI;
1522 while (reg >= VCPU_REGS_RAX) {
1523 if (reg == VCPU_REGS_RSP) {
1524 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1529 rc = emulate_pop(ctxt, ctxt->ops, &c->regs[reg], c->op_bytes);
1530 if (rc != X86EMUL_CONTINUE)
1537 int emulate_int_real(struct x86_emulate_ctxt *ctxt,
1538 struct x86_emulate_ops *ops, int irq)
1540 struct decode_cache *c = &ctxt->decode;
1547 /* TODO: Add limit checks */
1548 c->src.val = ctxt->eflags;
1550 if (rc != X86EMUL_CONTINUE)
1553 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1555 c->src.val = ops->get_segment_selector(ctxt, VCPU_SREG_CS);
1557 if (rc != X86EMUL_CONTINUE)
1560 c->src.val = c->eip;
1562 if (rc != X86EMUL_CONTINUE)
1565 ops->get_idt(ctxt, &dt);
1567 eip_addr = dt.address + (irq << 2);
1568 cs_addr = dt.address + (irq << 2) + 2;
1570 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1571 if (rc != X86EMUL_CONTINUE)
1574 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1575 if (rc != X86EMUL_CONTINUE)
1578 rc = load_segment_descriptor(ctxt, ops, cs, VCPU_SREG_CS);
1579 if (rc != X86EMUL_CONTINUE)
1587 static int emulate_int(struct x86_emulate_ctxt *ctxt,
1588 struct x86_emulate_ops *ops, int irq)
1590 switch(ctxt->mode) {
1591 case X86EMUL_MODE_REAL:
1592 return emulate_int_real(ctxt, ops, irq);
1593 case X86EMUL_MODE_VM86:
1594 case X86EMUL_MODE_PROT16:
1595 case X86EMUL_MODE_PROT32:
1596 case X86EMUL_MODE_PROT64:
1598 /* Protected mode interrupts unimplemented yet */
1599 return X86EMUL_UNHANDLEABLE;
1603 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
1604 struct x86_emulate_ops *ops)
1606 struct decode_cache *c = &ctxt->decode;
1607 int rc = X86EMUL_CONTINUE;
1608 unsigned long temp_eip = 0;
1609 unsigned long temp_eflags = 0;
1610 unsigned long cs = 0;
1611 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1612 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1613 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1614 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1616 /* TODO: Add stack limit check */
1618 rc = emulate_pop(ctxt, ops, &temp_eip, c->op_bytes);
1620 if (rc != X86EMUL_CONTINUE)
1623 if (temp_eip & ~0xffff)
1624 return emulate_gp(ctxt, 0);
1626 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1628 if (rc != X86EMUL_CONTINUE)
1631 rc = emulate_pop(ctxt, ops, &temp_eflags, c->op_bytes);
1633 if (rc != X86EMUL_CONTINUE)
1636 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1638 if (rc != X86EMUL_CONTINUE)
1644 if (c->op_bytes == 4)
1645 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1646 else if (c->op_bytes == 2) {
1647 ctxt->eflags &= ~0xffff;
1648 ctxt->eflags |= temp_eflags;
1651 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1652 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1657 static inline int emulate_iret(struct x86_emulate_ctxt *ctxt,
1658 struct x86_emulate_ops* ops)
1660 switch(ctxt->mode) {
1661 case X86EMUL_MODE_REAL:
1662 return emulate_iret_real(ctxt, ops);
1663 case X86EMUL_MODE_VM86:
1664 case X86EMUL_MODE_PROT16:
1665 case X86EMUL_MODE_PROT32:
1666 case X86EMUL_MODE_PROT64:
1668 /* iret from protected mode unimplemented yet */
1669 return X86EMUL_UNHANDLEABLE;
1673 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1674 struct x86_emulate_ops *ops)
1676 struct decode_cache *c = &ctxt->decode;
1678 return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1681 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1683 struct decode_cache *c = &ctxt->decode;
1684 switch (c->modrm_reg) {
1686 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1689 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1692 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1695 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1697 case 4: /* sal/shl */
1698 case 6: /* sal/shl */
1699 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1702 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1705 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1710 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1711 struct x86_emulate_ops *ops)
1713 struct decode_cache *c = &ctxt->decode;
1714 unsigned long *rax = &c->regs[VCPU_REGS_RAX];
1715 unsigned long *rdx = &c->regs[VCPU_REGS_RDX];
1718 switch (c->modrm_reg) {
1719 case 0 ... 1: /* test */
1720 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1723 c->dst.val = ~c->dst.val;
1726 emulate_1op("neg", c->dst, ctxt->eflags);
1729 emulate_1op_rax_rdx("mul", c->src, *rax, *rdx, ctxt->eflags);
1732 emulate_1op_rax_rdx("imul", c->src, *rax, *rdx, ctxt->eflags);
1735 emulate_1op_rax_rdx_ex("div", c->src, *rax, *rdx,
1739 emulate_1op_rax_rdx_ex("idiv", c->src, *rax, *rdx,
1743 return X86EMUL_UNHANDLEABLE;
1746 return emulate_de(ctxt);
1747 return X86EMUL_CONTINUE;
1750 static int emulate_grp45(struct x86_emulate_ctxt *ctxt)
1752 struct decode_cache *c = &ctxt->decode;
1753 int rc = X86EMUL_CONTINUE;
1755 switch (c->modrm_reg) {
1757 emulate_1op("inc", c->dst, ctxt->eflags);
1760 emulate_1op("dec", c->dst, ctxt->eflags);
1762 case 2: /* call near abs */ {
1765 c->eip = c->src.val;
1766 c->src.val = old_eip;
1770 case 4: /* jmp abs */
1771 c->eip = c->src.val;
1780 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1781 struct x86_emulate_ops *ops)
1783 struct decode_cache *c = &ctxt->decode;
1784 u64 old = c->dst.orig_val64;
1786 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1787 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1788 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1789 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1790 ctxt->eflags &= ~EFLG_ZF;
1792 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1793 (u32) c->regs[VCPU_REGS_RBX];
1795 ctxt->eflags |= EFLG_ZF;
1797 return X86EMUL_CONTINUE;
1800 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1801 struct x86_emulate_ops *ops)
1803 struct decode_cache *c = &ctxt->decode;
1807 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1808 if (rc != X86EMUL_CONTINUE)
1810 if (c->op_bytes == 4)
1811 c->eip = (u32)c->eip;
1812 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1813 if (rc != X86EMUL_CONTINUE)
1815 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1819 static int emulate_load_segment(struct x86_emulate_ctxt *ctxt,
1820 struct x86_emulate_ops *ops, int seg)
1822 struct decode_cache *c = &ctxt->decode;
1826 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
1828 rc = load_segment_descriptor(ctxt, ops, sel, seg);
1829 if (rc != X86EMUL_CONTINUE)
1832 c->dst.val = c->src.val;
1837 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1838 struct x86_emulate_ops *ops, struct desc_struct *cs,
1839 struct desc_struct *ss)
1841 memset(cs, 0, sizeof(struct desc_struct));
1842 ops->get_cached_descriptor(ctxt, cs, NULL, VCPU_SREG_CS);
1843 memset(ss, 0, sizeof(struct desc_struct));
1845 cs->l = 0; /* will be adjusted later */
1846 set_desc_base(cs, 0); /* flat segment */
1847 cs->g = 1; /* 4kb granularity */
1848 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1849 cs->type = 0x0b; /* Read, Execute, Accessed */
1851 cs->dpl = 0; /* will be adjusted later */
1855 set_desc_base(ss, 0); /* flat segment */
1856 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1857 ss->g = 1; /* 4kb granularity */
1859 ss->type = 0x03; /* Read/Write, Accessed */
1860 ss->d = 1; /* 32bit stack segment */
1866 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1868 struct decode_cache *c = &ctxt->decode;
1869 struct desc_struct cs, ss;
1874 /* syscall is not available in real mode */
1875 if (ctxt->mode == X86EMUL_MODE_REAL ||
1876 ctxt->mode == X86EMUL_MODE_VM86)
1877 return emulate_ud(ctxt);
1879 ops->get_msr(ctxt, MSR_EFER, &efer);
1880 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1882 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1884 cs_sel = (u16)(msr_data & 0xfffc);
1885 ss_sel = (u16)(msr_data + 8);
1887 if (efer & EFER_LMA) {
1891 ops->set_cached_descriptor(ctxt, &cs, 0, VCPU_SREG_CS);
1892 ops->set_segment_selector(ctxt, cs_sel, VCPU_SREG_CS);
1893 ops->set_cached_descriptor(ctxt, &ss, 0, VCPU_SREG_SS);
1894 ops->set_segment_selector(ctxt, ss_sel, VCPU_SREG_SS);
1896 c->regs[VCPU_REGS_RCX] = c->eip;
1897 if (efer & EFER_LMA) {
1898 #ifdef CONFIG_X86_64
1899 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1902 ctxt->mode == X86EMUL_MODE_PROT64 ?
1903 MSR_LSTAR : MSR_CSTAR, &msr_data);
1906 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1907 ctxt->eflags &= ~(msr_data | EFLG_RF);
1911 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1912 c->eip = (u32)msr_data;
1914 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1917 return X86EMUL_CONTINUE;
1921 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1923 struct decode_cache *c = &ctxt->decode;
1924 struct desc_struct cs, ss;
1929 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1930 /* inject #GP if in real mode */
1931 if (ctxt->mode == X86EMUL_MODE_REAL)
1932 return emulate_gp(ctxt, 0);
1934 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1935 * Therefore, we inject an #UD.
1937 if (ctxt->mode == X86EMUL_MODE_PROT64)
1938 return emulate_ud(ctxt);
1940 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1942 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1943 switch (ctxt->mode) {
1944 case X86EMUL_MODE_PROT32:
1945 if ((msr_data & 0xfffc) == 0x0)
1946 return emulate_gp(ctxt, 0);
1948 case X86EMUL_MODE_PROT64:
1949 if (msr_data == 0x0)
1950 return emulate_gp(ctxt, 0);
1954 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1955 cs_sel = (u16)msr_data;
1956 cs_sel &= ~SELECTOR_RPL_MASK;
1957 ss_sel = cs_sel + 8;
1958 ss_sel &= ~SELECTOR_RPL_MASK;
1959 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
1964 ops->set_cached_descriptor(ctxt, &cs, 0, VCPU_SREG_CS);
1965 ops->set_segment_selector(ctxt, cs_sel, VCPU_SREG_CS);
1966 ops->set_cached_descriptor(ctxt, &ss, 0, VCPU_SREG_SS);
1967 ops->set_segment_selector(ctxt, ss_sel, VCPU_SREG_SS);
1969 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
1972 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
1973 c->regs[VCPU_REGS_RSP] = msr_data;
1975 return X86EMUL_CONTINUE;
1979 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1981 struct decode_cache *c = &ctxt->decode;
1982 struct desc_struct cs, ss;
1987 /* inject #GP if in real mode or Virtual 8086 mode */
1988 if (ctxt->mode == X86EMUL_MODE_REAL ||
1989 ctxt->mode == X86EMUL_MODE_VM86)
1990 return emulate_gp(ctxt, 0);
1992 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1994 if ((c->rex_prefix & 0x8) != 0x0)
1995 usermode = X86EMUL_MODE_PROT64;
1997 usermode = X86EMUL_MODE_PROT32;
2001 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2003 case X86EMUL_MODE_PROT32:
2004 cs_sel = (u16)(msr_data + 16);
2005 if ((msr_data & 0xfffc) == 0x0)
2006 return emulate_gp(ctxt, 0);
2007 ss_sel = (u16)(msr_data + 24);
2009 case X86EMUL_MODE_PROT64:
2010 cs_sel = (u16)(msr_data + 32);
2011 if (msr_data == 0x0)
2012 return emulate_gp(ctxt, 0);
2013 ss_sel = cs_sel + 8;
2018 cs_sel |= SELECTOR_RPL_MASK;
2019 ss_sel |= SELECTOR_RPL_MASK;
2021 ops->set_cached_descriptor(ctxt, &cs, 0, VCPU_SREG_CS);
2022 ops->set_segment_selector(ctxt, cs_sel, VCPU_SREG_CS);
2023 ops->set_cached_descriptor(ctxt, &ss, 0, VCPU_SREG_SS);
2024 ops->set_segment_selector(ctxt, ss_sel, VCPU_SREG_SS);
2026 c->eip = c->regs[VCPU_REGS_RDX];
2027 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
2029 return X86EMUL_CONTINUE;
2032 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
2033 struct x86_emulate_ops *ops)
2036 if (ctxt->mode == X86EMUL_MODE_REAL)
2038 if (ctxt->mode == X86EMUL_MODE_VM86)
2040 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2041 return ops->cpl(ctxt) > iopl;
2044 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2045 struct x86_emulate_ops *ops,
2048 struct desc_struct tr_seg;
2051 u16 io_bitmap_ptr, perm, bit_idx = port & 0x7;
2052 unsigned mask = (1 << len) - 1;
2055 ops->get_cached_descriptor(ctxt, &tr_seg, &base3, VCPU_SREG_TR);
2058 if (desc_limit_scaled(&tr_seg) < 103)
2060 base = get_desc_base(&tr_seg);
2061 #ifdef CONFIG_X86_64
2062 base |= ((u64)base3) << 32;
2064 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2065 if (r != X86EMUL_CONTINUE)
2067 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2069 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2070 if (r != X86EMUL_CONTINUE)
2072 if ((perm >> bit_idx) & mask)
2077 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2078 struct x86_emulate_ops *ops,
2084 if (emulator_bad_iopl(ctxt, ops))
2085 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
2088 ctxt->perm_ok = true;
2093 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2094 struct x86_emulate_ops *ops,
2095 struct tss_segment_16 *tss)
2097 struct decode_cache *c = &ctxt->decode;
2100 tss->flag = ctxt->eflags;
2101 tss->ax = c->regs[VCPU_REGS_RAX];
2102 tss->cx = c->regs[VCPU_REGS_RCX];
2103 tss->dx = c->regs[VCPU_REGS_RDX];
2104 tss->bx = c->regs[VCPU_REGS_RBX];
2105 tss->sp = c->regs[VCPU_REGS_RSP];
2106 tss->bp = c->regs[VCPU_REGS_RBP];
2107 tss->si = c->regs[VCPU_REGS_RSI];
2108 tss->di = c->regs[VCPU_REGS_RDI];
2110 tss->es = ops->get_segment_selector(ctxt, VCPU_SREG_ES);
2111 tss->cs = ops->get_segment_selector(ctxt, VCPU_SREG_CS);
2112 tss->ss = ops->get_segment_selector(ctxt, VCPU_SREG_SS);
2113 tss->ds = ops->get_segment_selector(ctxt, VCPU_SREG_DS);
2114 tss->ldt = ops->get_segment_selector(ctxt, VCPU_SREG_LDTR);
2117 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2118 struct x86_emulate_ops *ops,
2119 struct tss_segment_16 *tss)
2121 struct decode_cache *c = &ctxt->decode;
2125 ctxt->eflags = tss->flag | 2;
2126 c->regs[VCPU_REGS_RAX] = tss->ax;
2127 c->regs[VCPU_REGS_RCX] = tss->cx;
2128 c->regs[VCPU_REGS_RDX] = tss->dx;
2129 c->regs[VCPU_REGS_RBX] = tss->bx;
2130 c->regs[VCPU_REGS_RSP] = tss->sp;
2131 c->regs[VCPU_REGS_RBP] = tss->bp;
2132 c->regs[VCPU_REGS_RSI] = tss->si;
2133 c->regs[VCPU_REGS_RDI] = tss->di;
2136 * SDM says that segment selectors are loaded before segment
2139 ops->set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2140 ops->set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2141 ops->set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2142 ops->set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2143 ops->set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2146 * Now load segment descriptors. If fault happenes at this stage
2147 * it is handled in a context of new task
2149 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
2150 if (ret != X86EMUL_CONTINUE)
2152 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2153 if (ret != X86EMUL_CONTINUE)
2155 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2156 if (ret != X86EMUL_CONTINUE)
2158 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2159 if (ret != X86EMUL_CONTINUE)
2161 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2162 if (ret != X86EMUL_CONTINUE)
2165 return X86EMUL_CONTINUE;
2168 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2169 struct x86_emulate_ops *ops,
2170 u16 tss_selector, u16 old_tss_sel,
2171 ulong old_tss_base, struct desc_struct *new_desc)
2173 struct tss_segment_16 tss_seg;
2175 u32 new_tss_base = get_desc_base(new_desc);
2177 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2179 if (ret != X86EMUL_CONTINUE)
2180 /* FIXME: need to provide precise fault address */
2183 save_state_to_tss16(ctxt, ops, &tss_seg);
2185 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2187 if (ret != X86EMUL_CONTINUE)
2188 /* FIXME: need to provide precise fault address */
2191 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2193 if (ret != X86EMUL_CONTINUE)
2194 /* FIXME: need to provide precise fault address */
2197 if (old_tss_sel != 0xffff) {
2198 tss_seg.prev_task_link = old_tss_sel;
2200 ret = ops->write_std(ctxt, new_tss_base,
2201 &tss_seg.prev_task_link,
2202 sizeof tss_seg.prev_task_link,
2204 if (ret != X86EMUL_CONTINUE)
2205 /* FIXME: need to provide precise fault address */
2209 return load_state_from_tss16(ctxt, ops, &tss_seg);
2212 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2213 struct x86_emulate_ops *ops,
2214 struct tss_segment_32 *tss)
2216 struct decode_cache *c = &ctxt->decode;
2218 tss->cr3 = ops->get_cr(ctxt, 3);
2220 tss->eflags = ctxt->eflags;
2221 tss->eax = c->regs[VCPU_REGS_RAX];
2222 tss->ecx = c->regs[VCPU_REGS_RCX];
2223 tss->edx = c->regs[VCPU_REGS_RDX];
2224 tss->ebx = c->regs[VCPU_REGS_RBX];
2225 tss->esp = c->regs[VCPU_REGS_RSP];
2226 tss->ebp = c->regs[VCPU_REGS_RBP];
2227 tss->esi = c->regs[VCPU_REGS_RSI];
2228 tss->edi = c->regs[VCPU_REGS_RDI];
2230 tss->es = ops->get_segment_selector(ctxt, VCPU_SREG_ES);
2231 tss->cs = ops->get_segment_selector(ctxt, VCPU_SREG_CS);
2232 tss->ss = ops->get_segment_selector(ctxt, VCPU_SREG_SS);
2233 tss->ds = ops->get_segment_selector(ctxt, VCPU_SREG_DS);
2234 tss->fs = ops->get_segment_selector(ctxt, VCPU_SREG_FS);
2235 tss->gs = ops->get_segment_selector(ctxt, VCPU_SREG_GS);
2236 tss->ldt_selector = ops->get_segment_selector(ctxt, VCPU_SREG_LDTR);
2239 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2240 struct x86_emulate_ops *ops,
2241 struct tss_segment_32 *tss)
2243 struct decode_cache *c = &ctxt->decode;
2246 if (ops->set_cr(ctxt, 3, tss->cr3))
2247 return emulate_gp(ctxt, 0);
2249 ctxt->eflags = tss->eflags | 2;
2250 c->regs[VCPU_REGS_RAX] = tss->eax;
2251 c->regs[VCPU_REGS_RCX] = tss->ecx;
2252 c->regs[VCPU_REGS_RDX] = tss->edx;
2253 c->regs[VCPU_REGS_RBX] = tss->ebx;
2254 c->regs[VCPU_REGS_RSP] = tss->esp;
2255 c->regs[VCPU_REGS_RBP] = tss->ebp;
2256 c->regs[VCPU_REGS_RSI] = tss->esi;
2257 c->regs[VCPU_REGS_RDI] = tss->edi;
2260 * SDM says that segment selectors are loaded before segment
2263 ops->set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2264 ops->set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2265 ops->set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2266 ops->set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2267 ops->set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2268 ops->set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2269 ops->set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2272 * Now load segment descriptors. If fault happenes at this stage
2273 * it is handled in a context of new task
2275 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2276 if (ret != X86EMUL_CONTINUE)
2278 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2279 if (ret != X86EMUL_CONTINUE)
2281 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2282 if (ret != X86EMUL_CONTINUE)
2284 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2285 if (ret != X86EMUL_CONTINUE)
2287 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2288 if (ret != X86EMUL_CONTINUE)
2290 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2291 if (ret != X86EMUL_CONTINUE)
2293 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2294 if (ret != X86EMUL_CONTINUE)
2297 return X86EMUL_CONTINUE;
2300 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2301 struct x86_emulate_ops *ops,
2302 u16 tss_selector, u16 old_tss_sel,
2303 ulong old_tss_base, struct desc_struct *new_desc)
2305 struct tss_segment_32 tss_seg;
2307 u32 new_tss_base = get_desc_base(new_desc);
2309 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2311 if (ret != X86EMUL_CONTINUE)
2312 /* FIXME: need to provide precise fault address */
2315 save_state_to_tss32(ctxt, ops, &tss_seg);
2317 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2319 if (ret != X86EMUL_CONTINUE)
2320 /* FIXME: need to provide precise fault address */
2323 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2325 if (ret != X86EMUL_CONTINUE)
2326 /* FIXME: need to provide precise fault address */
2329 if (old_tss_sel != 0xffff) {
2330 tss_seg.prev_task_link = old_tss_sel;
2332 ret = ops->write_std(ctxt, new_tss_base,
2333 &tss_seg.prev_task_link,
2334 sizeof tss_seg.prev_task_link,
2336 if (ret != X86EMUL_CONTINUE)
2337 /* FIXME: need to provide precise fault address */
2341 return load_state_from_tss32(ctxt, ops, &tss_seg);
2344 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2345 struct x86_emulate_ops *ops,
2346 u16 tss_selector, int reason,
2347 bool has_error_code, u32 error_code)
2349 struct desc_struct curr_tss_desc, next_tss_desc;
2351 u16 old_tss_sel = ops->get_segment_selector(ctxt, VCPU_SREG_TR);
2352 ulong old_tss_base =
2353 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2356 /* FIXME: old_tss_base == ~0 ? */
2358 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2359 if (ret != X86EMUL_CONTINUE)
2361 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2362 if (ret != X86EMUL_CONTINUE)
2365 /* FIXME: check that next_tss_desc is tss */
2367 if (reason != TASK_SWITCH_IRET) {
2368 if ((tss_selector & 3) > next_tss_desc.dpl ||
2369 ops->cpl(ctxt) > next_tss_desc.dpl)
2370 return emulate_gp(ctxt, 0);
2373 desc_limit = desc_limit_scaled(&next_tss_desc);
2374 if (!next_tss_desc.p ||
2375 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2376 desc_limit < 0x2b)) {
2377 emulate_ts(ctxt, tss_selector & 0xfffc);
2378 return X86EMUL_PROPAGATE_FAULT;
2381 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2382 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2383 write_segment_descriptor(ctxt, ops, old_tss_sel,
2387 if (reason == TASK_SWITCH_IRET)
2388 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2390 /* set back link to prev task only if NT bit is set in eflags
2391 note that old_tss_sel is not used afetr this point */
2392 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2393 old_tss_sel = 0xffff;
2395 if (next_tss_desc.type & 8)
2396 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2397 old_tss_base, &next_tss_desc);
2399 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2400 old_tss_base, &next_tss_desc);
2401 if (ret != X86EMUL_CONTINUE)
2404 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2405 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2407 if (reason != TASK_SWITCH_IRET) {
2408 next_tss_desc.type |= (1 << 1); /* set busy flag */
2409 write_segment_descriptor(ctxt, ops, tss_selector,
2413 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2414 ops->set_cached_descriptor(ctxt, &next_tss_desc, 0, VCPU_SREG_TR);
2415 ops->set_segment_selector(ctxt, tss_selector, VCPU_SREG_TR);
2417 if (has_error_code) {
2418 struct decode_cache *c = &ctxt->decode;
2420 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2422 c->src.val = (unsigned long) error_code;
2423 ret = em_push(ctxt);
2429 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2430 u16 tss_selector, int reason,
2431 bool has_error_code, u32 error_code)
2433 struct x86_emulate_ops *ops = ctxt->ops;
2434 struct decode_cache *c = &ctxt->decode;
2438 c->dst.type = OP_NONE;
2440 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2441 has_error_code, error_code);
2443 if (rc == X86EMUL_CONTINUE)
2446 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2449 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2450 int reg, struct operand *op)
2452 struct decode_cache *c = &ctxt->decode;
2453 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2455 register_address_increment(c, &c->regs[reg], df * op->bytes);
2456 op->addr.mem.ea = register_address(c, c->regs[reg]);
2457 op->addr.mem.seg = seg;
2460 static int em_das(struct x86_emulate_ctxt *ctxt)
2462 struct decode_cache *c = &ctxt->decode;
2464 bool af, cf, old_cf;
2466 cf = ctxt->eflags & X86_EFLAGS_CF;
2472 af = ctxt->eflags & X86_EFLAGS_AF;
2473 if ((al & 0x0f) > 9 || af) {
2475 cf = old_cf | (al >= 250);
2480 if (old_al > 0x99 || old_cf) {
2486 /* Set PF, ZF, SF */
2487 c->src.type = OP_IMM;
2490 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2491 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2493 ctxt->eflags |= X86_EFLAGS_CF;
2495 ctxt->eflags |= X86_EFLAGS_AF;
2496 return X86EMUL_CONTINUE;
2499 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2501 struct decode_cache *c = &ctxt->decode;
2506 old_cs = ctxt->ops->get_segment_selector(ctxt, VCPU_SREG_CS);
2509 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
2510 if (load_segment_descriptor(ctxt, ctxt->ops, sel, VCPU_SREG_CS))
2511 return X86EMUL_CONTINUE;
2514 memcpy(&c->eip, c->src.valptr, c->op_bytes);
2516 c->src.val = old_cs;
2518 if (rc != X86EMUL_CONTINUE)
2521 c->src.val = old_eip;
2522 return em_push(ctxt);
2525 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2527 struct decode_cache *c = &ctxt->decode;
2530 c->dst.type = OP_REG;
2531 c->dst.addr.reg = &c->eip;
2532 c->dst.bytes = c->op_bytes;
2533 rc = emulate_pop(ctxt, ctxt->ops, &c->dst.val, c->op_bytes);
2534 if (rc != X86EMUL_CONTINUE)
2536 register_address_increment(c, &c->regs[VCPU_REGS_RSP], c->src.val);
2537 return X86EMUL_CONTINUE;
2540 static int em_add(struct x86_emulate_ctxt *ctxt)
2542 struct decode_cache *c = &ctxt->decode;
2544 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
2545 return X86EMUL_CONTINUE;
2548 static int em_or(struct x86_emulate_ctxt *ctxt)
2550 struct decode_cache *c = &ctxt->decode;
2552 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2553 return X86EMUL_CONTINUE;
2556 static int em_adc(struct x86_emulate_ctxt *ctxt)
2558 struct decode_cache *c = &ctxt->decode;
2560 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
2561 return X86EMUL_CONTINUE;
2564 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2566 struct decode_cache *c = &ctxt->decode;
2568 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
2569 return X86EMUL_CONTINUE;
2572 static int em_and(struct x86_emulate_ctxt *ctxt)
2574 struct decode_cache *c = &ctxt->decode;
2576 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
2577 return X86EMUL_CONTINUE;
2580 static int em_sub(struct x86_emulate_ctxt *ctxt)
2582 struct decode_cache *c = &ctxt->decode;
2584 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
2585 return X86EMUL_CONTINUE;
2588 static int em_xor(struct x86_emulate_ctxt *ctxt)
2590 struct decode_cache *c = &ctxt->decode;
2592 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
2593 return X86EMUL_CONTINUE;
2596 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2598 struct decode_cache *c = &ctxt->decode;
2600 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2601 /* Disable writeback. */
2602 c->dst.type = OP_NONE;
2603 return X86EMUL_CONTINUE;
2606 static int em_imul(struct x86_emulate_ctxt *ctxt)
2608 struct decode_cache *c = &ctxt->decode;
2610 emulate_2op_SrcV_nobyte("imul", c->src, c->dst, ctxt->eflags);
2611 return X86EMUL_CONTINUE;
2614 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2616 struct decode_cache *c = &ctxt->decode;
2618 c->dst.val = c->src2.val;
2619 return em_imul(ctxt);
2622 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2624 struct decode_cache *c = &ctxt->decode;
2626 c->dst.type = OP_REG;
2627 c->dst.bytes = c->src.bytes;
2628 c->dst.addr.reg = &c->regs[VCPU_REGS_RDX];
2629 c->dst.val = ~((c->src.val >> (c->src.bytes * 8 - 1)) - 1);
2631 return X86EMUL_CONTINUE;
2634 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2636 struct decode_cache *c = &ctxt->decode;
2639 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2640 c->regs[VCPU_REGS_RAX] = (u32)tsc;
2641 c->regs[VCPU_REGS_RDX] = tsc >> 32;
2642 return X86EMUL_CONTINUE;
2645 static int em_mov(struct x86_emulate_ctxt *ctxt)
2647 struct decode_cache *c = &ctxt->decode;
2648 c->dst.val = c->src.val;
2649 return X86EMUL_CONTINUE;
2652 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2654 struct decode_cache *c = &ctxt->decode;
2655 memcpy(&c->dst.vec_val, &c->src.vec_val, c->op_bytes);
2656 return X86EMUL_CONTINUE;
2659 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2661 struct decode_cache *c = &ctxt->decode;
2665 rc = linearize(ctxt, c->src.addr.mem, 1, false, &linear);
2666 if (rc == X86EMUL_CONTINUE)
2667 ctxt->ops->invlpg(ctxt, linear);
2668 /* Disable writeback. */
2669 c->dst.type = OP_NONE;
2670 return X86EMUL_CONTINUE;
2673 static int em_clts(struct x86_emulate_ctxt *ctxt)
2677 cr0 = ctxt->ops->get_cr(ctxt, 0);
2679 ctxt->ops->set_cr(ctxt, 0, cr0);
2680 return X86EMUL_CONTINUE;
2683 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
2685 struct decode_cache *c = &ctxt->decode;
2688 if (c->modrm_mod != 3 || c->modrm_rm != 1)
2689 return X86EMUL_UNHANDLEABLE;
2691 rc = ctxt->ops->fix_hypercall(ctxt);
2692 if (rc != X86EMUL_CONTINUE)
2695 /* Let the processor re-execute the fixed hypercall */
2697 /* Disable writeback. */
2698 c->dst.type = OP_NONE;
2699 return X86EMUL_CONTINUE;
2702 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
2704 struct decode_cache *c = &ctxt->decode;
2705 struct desc_ptr desc_ptr;
2708 rc = read_descriptor(ctxt, ctxt->ops, c->src.addr.mem,
2709 &desc_ptr.size, &desc_ptr.address,
2711 if (rc != X86EMUL_CONTINUE)
2713 ctxt->ops->set_gdt(ctxt, &desc_ptr);
2714 /* Disable writeback. */
2715 c->dst.type = OP_NONE;
2716 return X86EMUL_CONTINUE;
2719 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2721 struct decode_cache *c = &ctxt->decode;
2724 rc = ctxt->ops->fix_hypercall(ctxt);
2726 /* Disable writeback. */
2727 c->dst.type = OP_NONE;
2731 static int em_lidt(struct x86_emulate_ctxt *ctxt)
2733 struct decode_cache *c = &ctxt->decode;
2734 struct desc_ptr desc_ptr;
2737 rc = read_descriptor(ctxt, ctxt->ops, c->src.addr.mem,
2741 if (rc != X86EMUL_CONTINUE)
2743 ctxt->ops->set_idt(ctxt, &desc_ptr);
2744 /* Disable writeback. */
2745 c->dst.type = OP_NONE;
2746 return X86EMUL_CONTINUE;
2749 static int em_smsw(struct x86_emulate_ctxt *ctxt)
2751 struct decode_cache *c = &ctxt->decode;
2754 c->dst.val = ctxt->ops->get_cr(ctxt, 0);
2755 return X86EMUL_CONTINUE;
2758 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
2760 struct decode_cache *c = &ctxt->decode;
2761 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2762 | (c->src.val & 0x0f));
2763 c->dst.type = OP_NONE;
2764 return X86EMUL_CONTINUE;
2767 static bool valid_cr(int nr)
2779 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2781 struct decode_cache *c = &ctxt->decode;
2783 if (!valid_cr(c->modrm_reg))
2784 return emulate_ud(ctxt);
2786 return X86EMUL_CONTINUE;
2789 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2791 struct decode_cache *c = &ctxt->decode;
2792 u64 new_val = c->src.val64;
2793 int cr = c->modrm_reg;
2796 static u64 cr_reserved_bits[] = {
2797 0xffffffff00000000ULL,
2798 0, 0, 0, /* CR3 checked later */
2805 return emulate_ud(ctxt);
2807 if (new_val & cr_reserved_bits[cr])
2808 return emulate_gp(ctxt, 0);
2813 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2814 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2815 return emulate_gp(ctxt, 0);
2817 cr4 = ctxt->ops->get_cr(ctxt, 4);
2818 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2820 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2821 !(cr4 & X86_CR4_PAE))
2822 return emulate_gp(ctxt, 0);
2829 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2830 if (efer & EFER_LMA)
2831 rsvd = CR3_L_MODE_RESERVED_BITS;
2832 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
2833 rsvd = CR3_PAE_RESERVED_BITS;
2834 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
2835 rsvd = CR3_NONPAE_RESERVED_BITS;
2838 return emulate_gp(ctxt, 0);
2845 cr4 = ctxt->ops->get_cr(ctxt, 4);
2846 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2848 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2849 return emulate_gp(ctxt, 0);
2855 return X86EMUL_CONTINUE;
2858 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2862 ctxt->ops->get_dr(ctxt, 7, &dr7);
2864 /* Check if DR7.Global_Enable is set */
2865 return dr7 & (1 << 13);
2868 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2870 struct decode_cache *c = &ctxt->decode;
2871 int dr = c->modrm_reg;
2875 return emulate_ud(ctxt);
2877 cr4 = ctxt->ops->get_cr(ctxt, 4);
2878 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2879 return emulate_ud(ctxt);
2881 if (check_dr7_gd(ctxt))
2882 return emulate_db(ctxt);
2884 return X86EMUL_CONTINUE;
2887 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2889 struct decode_cache *c = &ctxt->decode;
2890 u64 new_val = c->src.val64;
2891 int dr = c->modrm_reg;
2893 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2894 return emulate_gp(ctxt, 0);
2896 return check_dr_read(ctxt);
2899 static int check_svme(struct x86_emulate_ctxt *ctxt)
2903 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2905 if (!(efer & EFER_SVME))
2906 return emulate_ud(ctxt);
2908 return X86EMUL_CONTINUE;
2911 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
2913 u64 rax = ctxt->decode.regs[VCPU_REGS_RAX];
2915 /* Valid physical address? */
2916 if (rax & 0xffff000000000000ULL)
2917 return emulate_gp(ctxt, 0);
2919 return check_svme(ctxt);
2922 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
2924 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2926 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
2927 return emulate_ud(ctxt);
2929 return X86EMUL_CONTINUE;
2932 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
2934 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2935 u64 rcx = ctxt->decode.regs[VCPU_REGS_RCX];
2937 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
2939 return emulate_gp(ctxt, 0);
2941 return X86EMUL_CONTINUE;
2944 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
2946 struct decode_cache *c = &ctxt->decode;
2948 c->dst.bytes = min(c->dst.bytes, 4u);
2949 if (!emulator_io_permited(ctxt, ctxt->ops, c->src.val, c->dst.bytes))
2950 return emulate_gp(ctxt, 0);
2952 return X86EMUL_CONTINUE;
2955 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
2957 struct decode_cache *c = &ctxt->decode;
2959 c->src.bytes = min(c->src.bytes, 4u);
2960 if (!emulator_io_permited(ctxt, ctxt->ops, c->dst.val, c->src.bytes))
2961 return emulate_gp(ctxt, 0);
2963 return X86EMUL_CONTINUE;
2966 #define D(_y) { .flags = (_y) }
2967 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2968 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
2969 .check_perm = (_p) }
2971 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
2972 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2973 #define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
2974 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2975 #define II(_f, _e, _i) \
2976 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2977 #define IIP(_f, _e, _i, _p) \
2978 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
2979 .check_perm = (_p) }
2980 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
2982 #define D2bv(_f) D((_f) | ByteOp), D(_f)
2983 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
2984 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
2986 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
2987 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
2988 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
2990 static struct opcode group7_rm1[] = {
2991 DI(SrcNone | ModRM | Priv, monitor),
2992 DI(SrcNone | ModRM | Priv, mwait),
2996 static struct opcode group7_rm3[] = {
2997 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
2998 II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
2999 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
3000 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
3001 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
3002 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
3003 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
3004 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
3007 static struct opcode group7_rm7[] = {
3009 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
3013 static struct opcode group1[] = {
3024 static struct opcode group1A[] = {
3025 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
3028 static struct opcode group3[] = {
3029 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
3030 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3031 X4(D(SrcMem | ModRM)),
3034 static struct opcode group4[] = {
3035 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
3039 static struct opcode group5[] = {
3040 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3041 D(SrcMem | ModRM | Stack),
3042 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3043 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
3044 D(SrcMem | ModRM | Stack), N,
3047 static struct opcode group6[] = {
3048 DI(ModRM | Prot, sldt),
3049 DI(ModRM | Prot, str),
3050 DI(ModRM | Prot | Priv, lldt),
3051 DI(ModRM | Prot | Priv, ltr),
3055 static struct group_dual group7 = { {
3056 DI(ModRM | Mov | DstMem | Priv, sgdt),
3057 DI(ModRM | Mov | DstMem | Priv, sidt),
3058 II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
3059 II(ModRM | SrcMem | Priv, em_lidt, lidt),
3060 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3061 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
3062 II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3064 I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
3066 N, EXT(0, group7_rm3),
3067 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3068 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3071 static struct opcode group8[] = {
3073 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
3074 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
3077 static struct group_dual group9 = { {
3078 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
3080 N, N, N, N, N, N, N, N,
3083 static struct opcode group11[] = {
3084 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
3087 static struct gprefix pfx_0f_6f_0f_7f = {
3088 N, N, N, I(Sse, em_movdqu),
3091 static struct opcode opcode_table[256] = {
3093 I6ALU(Lock, em_add),
3094 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3097 D(ImplicitOps | Stack | No64), N,
3099 I6ALU(Lock, em_adc),
3100 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3102 I6ALU(Lock, em_sbb),
3103 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3105 I6ALU(Lock, em_and), N, N,
3107 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3109 I6ALU(Lock, em_xor), N, N,
3111 I6ALU(0, em_cmp), N, N,
3115 X8(I(SrcReg | Stack, em_push)),
3117 X8(I(DstReg | Stack, em_pop)),
3119 I(ImplicitOps | Stack | No64, em_pusha),
3120 I(ImplicitOps | Stack | No64, em_popa),
3121 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3124 I(SrcImm | Mov | Stack, em_push),
3125 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3126 I(SrcImmByte | Mov | Stack, em_push),
3127 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3128 D2bvIP(DstDI | Mov | String, ins, check_perm_in), /* insb, insw/insd */
3129 D2bvIP(SrcSI | ImplicitOps | String, outs, check_perm_out), /* outsb, outsw/outsd */
3133 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
3134 G(DstMem | SrcImm | ModRM | Group, group1),
3135 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
3136 G(DstMem | SrcImmByte | ModRM | Group, group1),
3137 D2bv(DstMem | SrcReg | ModRM), D2bv(DstMem | SrcReg | ModRM | Lock),
3139 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
3140 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3141 D(DstMem | SrcNone | ModRM | Mov), D(ModRM | SrcMem | NoAccess | DstReg),
3142 D(ImplicitOps | SrcMem16 | ModRM), G(0, group1A),
3144 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3146 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3147 I(SrcImmFAddr | No64, em_call_far), N,
3148 II(ImplicitOps | Stack, em_pushf, pushf),
3149 II(ImplicitOps | Stack, em_popf, popf), N, N,
3151 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3152 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
3153 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3154 I2bv(SrcSI | DstDI | String, em_cmp),
3156 D2bv(DstAcc | SrcImm),
3157 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3158 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3159 I2bv(SrcAcc | DstDI | String, em_cmp),
3161 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3163 X8(I(DstReg | SrcImm | Mov, em_mov)),
3165 D2bv(DstMem | SrcImmByte | ModRM),
3166 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3167 D(ImplicitOps | Stack),
3168 D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
3169 G(ByteOp, group11), G(0, group11),
3171 N, N, N, D(ImplicitOps | Stack),
3172 D(ImplicitOps), DI(SrcImmByte, intn),
3173 D(ImplicitOps | No64), DI(ImplicitOps, iret),
3175 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3178 N, N, N, N, N, N, N, N,
3181 D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
3182 D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
3184 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
3185 D(SrcImmFAddr | No64), D(SrcImmByte | ImplicitOps),
3186 D2bvIP(SrcNone | DstAcc, in, check_perm_in),
3187 D2bvIP(SrcAcc | ImplicitOps, out, check_perm_out),
3189 N, DI(ImplicitOps, icebp), N, N,
3190 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3191 G(ByteOp, group3), G(0, group3),
3193 D(ImplicitOps), D(ImplicitOps), D(ImplicitOps), D(ImplicitOps),
3194 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3197 static struct opcode twobyte_table[256] = {
3199 G(0, group6), GD(0, &group7), N, N,
3200 N, D(ImplicitOps | VendorSpecific), DI(ImplicitOps | Priv, clts), N,
3201 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3202 N, D(ImplicitOps | ModRM), N, N,
3204 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3206 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3207 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3208 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3209 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3211 N, N, N, N, N, N, N, N,
3213 DI(ImplicitOps | Priv, wrmsr),
3214 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3215 DI(ImplicitOps | Priv, rdmsr),
3216 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3217 D(ImplicitOps | VendorSpecific), D(ImplicitOps | Priv | VendorSpecific),
3219 N, N, N, N, N, N, N, N,
3221 X16(D(DstReg | SrcMem | ModRM | Mov)),
3223 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3228 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3233 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3237 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3239 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3240 DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3241 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3242 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3244 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3245 DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3246 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3247 D(DstMem | SrcReg | Src2CL | ModRM),
3248 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3250 D2bv(DstMem | SrcReg | ModRM | Lock),
3251 D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3252 D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
3253 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3256 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3257 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3258 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3260 D2bv(DstMem | SrcReg | ModRM | Lock),
3261 N, D(DstMem | SrcReg | ModRM | Mov),
3262 N, N, N, GD(0, &group9),
3263 N, N, N, N, N, N, N, N,
3265 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3267 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3269 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3285 static unsigned imm_size(struct decode_cache *c)
3289 size = (c->d & ByteOp) ? 1 : c->op_bytes;
3295 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3296 unsigned size, bool sign_extension)
3298 struct decode_cache *c = &ctxt->decode;
3299 struct x86_emulate_ops *ops = ctxt->ops;
3300 int rc = X86EMUL_CONTINUE;
3304 op->addr.mem.ea = c->eip;
3305 /* NB. Immediates are sign-extended as necessary. */
3306 switch (op->bytes) {
3308 op->val = insn_fetch(s8, 1, c->eip);
3311 op->val = insn_fetch(s16, 2, c->eip);
3314 op->val = insn_fetch(s32, 4, c->eip);
3317 if (!sign_extension) {
3318 switch (op->bytes) {
3326 op->val &= 0xffffffff;
3335 x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3337 struct x86_emulate_ops *ops = ctxt->ops;
3338 struct decode_cache *c = &ctxt->decode;
3339 int rc = X86EMUL_CONTINUE;
3340 int mode = ctxt->mode;
3341 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3342 bool op_prefix = false;
3343 struct opcode opcode;
3344 struct operand memop = { .type = OP_NONE };
3347 c->fetch.start = c->eip;
3348 c->fetch.end = c->fetch.start + insn_len;
3350 memcpy(c->fetch.data, insn, insn_len);
3353 case X86EMUL_MODE_REAL:
3354 case X86EMUL_MODE_VM86:
3355 case X86EMUL_MODE_PROT16:
3356 def_op_bytes = def_ad_bytes = 2;
3358 case X86EMUL_MODE_PROT32:
3359 def_op_bytes = def_ad_bytes = 4;
3361 #ifdef CONFIG_X86_64
3362 case X86EMUL_MODE_PROT64:
3371 c->op_bytes = def_op_bytes;
3372 c->ad_bytes = def_ad_bytes;
3374 /* Legacy prefixes. */
3376 switch (c->b = insn_fetch(u8, 1, c->eip)) {
3377 case 0x66: /* operand-size override */
3379 /* switch between 2/4 bytes */
3380 c->op_bytes = def_op_bytes ^ 6;
3382 case 0x67: /* address-size override */
3383 if (mode == X86EMUL_MODE_PROT64)
3384 /* switch between 4/8 bytes */
3385 c->ad_bytes = def_ad_bytes ^ 12;
3387 /* switch between 2/4 bytes */
3388 c->ad_bytes = def_ad_bytes ^ 6;
3390 case 0x26: /* ES override */
3391 case 0x2e: /* CS override */
3392 case 0x36: /* SS override */
3393 case 0x3e: /* DS override */
3394 set_seg_override(c, (c->b >> 3) & 3);
3396 case 0x64: /* FS override */
3397 case 0x65: /* GS override */
3398 set_seg_override(c, c->b & 7);
3400 case 0x40 ... 0x4f: /* REX */
3401 if (mode != X86EMUL_MODE_PROT64)
3403 c->rex_prefix = c->b;
3405 case 0xf0: /* LOCK */
3408 case 0xf2: /* REPNE/REPNZ */
3409 case 0xf3: /* REP/REPE/REPZ */
3410 c->rep_prefix = c->b;
3416 /* Any legacy prefix after a REX prefix nullifies its effect. */
3424 if (c->rex_prefix & 8)
3425 c->op_bytes = 8; /* REX.W */
3427 /* Opcode byte(s). */
3428 opcode = opcode_table[c->b];
3429 /* Two-byte opcode? */
3432 c->b = insn_fetch(u8, 1, c->eip);
3433 opcode = twobyte_table[c->b];
3435 c->d = opcode.flags;
3437 while (c->d & GroupMask) {
3438 switch (c->d & GroupMask) {
3440 c->modrm = insn_fetch(u8, 1, c->eip);
3442 goffset = (c->modrm >> 3) & 7;
3443 opcode = opcode.u.group[goffset];
3446 c->modrm = insn_fetch(u8, 1, c->eip);
3448 goffset = (c->modrm >> 3) & 7;
3449 if ((c->modrm >> 6) == 3)
3450 opcode = opcode.u.gdual->mod3[goffset];
3452 opcode = opcode.u.gdual->mod012[goffset];
3455 goffset = c->modrm & 7;
3456 opcode = opcode.u.group[goffset];
3459 if (c->rep_prefix && op_prefix)
3460 return X86EMUL_UNHANDLEABLE;
3461 simd_prefix = op_prefix ? 0x66 : c->rep_prefix;
3462 switch (simd_prefix) {
3463 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3464 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3465 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3466 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3470 return X86EMUL_UNHANDLEABLE;
3474 c->d |= opcode.flags;
3477 c->execute = opcode.u.execute;
3478 c->check_perm = opcode.check_perm;
3479 c->intercept = opcode.intercept;
3482 if (c->d == 0 || (c->d & Undefined))
3485 if (!(c->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3488 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
3491 if (c->d & Op3264) {
3492 if (mode == X86EMUL_MODE_PROT64)
3501 /* ModRM and SIB bytes. */
3503 rc = decode_modrm(ctxt, ops, &memop);
3504 if (!c->has_seg_override)
3505 set_seg_override(c, c->modrm_seg);
3506 } else if (c->d & MemAbs)
3507 rc = decode_abs(ctxt, ops, &memop);
3508 if (rc != X86EMUL_CONTINUE)
3511 if (!c->has_seg_override)
3512 set_seg_override(c, VCPU_SREG_DS);
3514 memop.addr.mem.seg = seg_override(ctxt, ops, c);
3516 if (memop.type == OP_MEM && c->ad_bytes != 8)
3517 memop.addr.mem.ea = (u32)memop.addr.mem.ea;
3519 if (memop.type == OP_MEM && c->rip_relative)
3520 memop.addr.mem.ea += c->eip;
3523 * Decode and fetch the source operand: register, memory
3526 switch (c->d & SrcMask) {
3530 decode_register_operand(ctxt, &c->src, c, 0);
3539 memop.bytes = (c->d & ByteOp) ? 1 :
3545 rc = decode_imm(ctxt, &c->src, 2, false);
3548 rc = decode_imm(ctxt, &c->src, imm_size(c), true);
3551 rc = decode_imm(ctxt, &c->src, imm_size(c), false);
3554 rc = decode_imm(ctxt, &c->src, 1, true);
3557 rc = decode_imm(ctxt, &c->src, 1, false);
3560 c->src.type = OP_REG;
3561 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3562 c->src.addr.reg = &c->regs[VCPU_REGS_RAX];
3563 fetch_register_operand(&c->src);
3570 c->src.type = OP_MEM;
3571 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3572 c->src.addr.mem.ea =
3573 register_address(c, c->regs[VCPU_REGS_RSI]);
3574 c->src.addr.mem.seg = seg_override(ctxt, ops, c),
3578 c->src.type = OP_IMM;
3579 c->src.addr.mem.ea = c->eip;
3580 c->src.bytes = c->op_bytes + 2;
3581 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
3584 memop.bytes = c->op_bytes + 2;
3589 if (rc != X86EMUL_CONTINUE)
3593 * Decode and fetch the second source operand: register, memory
3596 switch (c->d & Src2Mask) {
3601 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
3604 rc = decode_imm(ctxt, &c->src2, 1, true);
3611 rc = decode_imm(ctxt, &c->src2, imm_size(c), true);
3615 if (rc != X86EMUL_CONTINUE)
3618 /* Decode and fetch the destination operand: register or memory. */
3619 switch (c->d & DstMask) {
3621 decode_register_operand(ctxt, &c->dst, c,
3622 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
3625 c->dst.type = OP_IMM;
3626 c->dst.addr.mem.ea = c->eip;
3628 c->dst.val = insn_fetch(u8, 1, c->eip);
3633 if ((c->d & DstMask) == DstMem64)
3636 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3638 fetch_bit_operand(c);
3639 c->dst.orig_val = c->dst.val;
3642 c->dst.type = OP_REG;
3643 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3644 c->dst.addr.reg = &c->regs[VCPU_REGS_RAX];
3645 fetch_register_operand(&c->dst);
3646 c->dst.orig_val = c->dst.val;
3649 c->dst.type = OP_MEM;
3650 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3651 c->dst.addr.mem.ea =
3652 register_address(c, c->regs[VCPU_REGS_RDI]);
3653 c->dst.addr.mem.seg = VCPU_SREG_ES;
3657 /* Special instructions do their own operand decoding. */
3659 c->dst.type = OP_NONE; /* Disable writeback. */
3664 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3667 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3669 struct decode_cache *c = &ctxt->decode;
3671 /* The second termination condition only applies for REPE
3672 * and REPNE. Test if the repeat string operation prefix is
3673 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3674 * corresponding termination condition according to:
3675 * - if REPE/REPZ and ZF = 0 then done
3676 * - if REPNE/REPNZ and ZF = 1 then done
3678 if (((c->b == 0xa6) || (c->b == 0xa7) ||
3679 (c->b == 0xae) || (c->b == 0xaf))
3680 && (((c->rep_prefix == REPE_PREFIX) &&
3681 ((ctxt->eflags & EFLG_ZF) == 0))
3682 || ((c->rep_prefix == REPNE_PREFIX) &&
3683 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3690 x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3692 struct x86_emulate_ops *ops = ctxt->ops;
3694 struct decode_cache *c = &ctxt->decode;
3695 int rc = X86EMUL_CONTINUE;
3696 int saved_dst_type = c->dst.type;
3697 int irq; /* Used for int 3, int, and into */
3699 ctxt->decode.mem_read.pos = 0;
3701 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
3702 rc = emulate_ud(ctxt);
3706 /* LOCK prefix is allowed only with some instructions */
3707 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
3708 rc = emulate_ud(ctxt);
3712 if ((c->d & SrcMask) == SrcMemFAddr && c->src.type != OP_MEM) {
3713 rc = emulate_ud(ctxt);
3718 && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
3719 || !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
3720 rc = emulate_ud(ctxt);
3724 if ((c->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
3725 rc = emulate_nm(ctxt);
3729 if (unlikely(ctxt->guest_mode) && c->intercept) {
3730 rc = emulator_check_intercept(ctxt, c->intercept,
3731 X86_ICPT_PRE_EXCEPT);
3732 if (rc != X86EMUL_CONTINUE)
3736 /* Privileged instruction can be executed only in CPL=0 */
3737 if ((c->d & Priv) && ops->cpl(ctxt)) {
3738 rc = emulate_gp(ctxt, 0);
3742 /* Instruction can only be executed in protected mode */
3743 if ((c->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3744 rc = emulate_ud(ctxt);
3748 /* Do instruction specific permission checks */
3749 if (c->check_perm) {
3750 rc = c->check_perm(ctxt);
3751 if (rc != X86EMUL_CONTINUE)
3755 if (unlikely(ctxt->guest_mode) && c->intercept) {
3756 rc = emulator_check_intercept(ctxt, c->intercept,
3757 X86_ICPT_POST_EXCEPT);
3758 if (rc != X86EMUL_CONTINUE)
3762 if (c->rep_prefix && (c->d & String)) {
3763 /* All REP prefixes have the same first termination condition */
3764 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
3770 if ((c->src.type == OP_MEM) && !(c->d & NoAccess)) {
3771 rc = segmented_read(ctxt, c->src.addr.mem,
3772 c->src.valptr, c->src.bytes);
3773 if (rc != X86EMUL_CONTINUE)
3775 c->src.orig_val64 = c->src.val64;
3778 if (c->src2.type == OP_MEM) {
3779 rc = segmented_read(ctxt, c->src2.addr.mem,
3780 &c->src2.val, c->src2.bytes);
3781 if (rc != X86EMUL_CONTINUE)
3785 if ((c->d & DstMask) == ImplicitOps)
3789 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
3790 /* optimisation - avoid slow emulated read if Mov */
3791 rc = segmented_read(ctxt, c->dst.addr.mem,
3792 &c->dst.val, c->dst.bytes);
3793 if (rc != X86EMUL_CONTINUE)
3796 c->dst.orig_val = c->dst.val;
3800 if (unlikely(ctxt->guest_mode) && c->intercept) {
3801 rc = emulator_check_intercept(ctxt, c->intercept,
3802 X86_ICPT_POST_MEMACCESS);
3803 if (rc != X86EMUL_CONTINUE)
3808 rc = c->execute(ctxt);
3809 if (rc != X86EMUL_CONTINUE)
3818 case 0x06: /* push es */
3819 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
3821 case 0x07: /* pop es */
3822 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
3824 case 0x0e: /* push cs */
3825 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
3827 case 0x16: /* push ss */
3828 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
3830 case 0x17: /* pop ss */
3831 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
3833 case 0x1e: /* push ds */
3834 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
3836 case 0x1f: /* pop ds */
3837 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
3839 case 0x40 ... 0x47: /* inc r16/r32 */
3840 emulate_1op("inc", c->dst, ctxt->eflags);
3842 case 0x48 ... 0x4f: /* dec r16/r32 */
3843 emulate_1op("dec", c->dst, ctxt->eflags);
3845 case 0x63: /* movsxd */
3846 if (ctxt->mode != X86EMUL_MODE_PROT64)
3847 goto cannot_emulate;
3848 c->dst.val = (s32) c->src.val;
3850 case 0x6c: /* insb */
3851 case 0x6d: /* insw/insd */
3852 c->src.val = c->regs[VCPU_REGS_RDX];
3854 case 0x6e: /* outsb */
3855 case 0x6f: /* outsw/outsd */
3856 c->dst.val = c->regs[VCPU_REGS_RDX];
3859 case 0x70 ... 0x7f: /* jcc (short) */
3860 if (test_cc(c->b, ctxt->eflags))
3861 jmp_rel(c, c->src.val);
3865 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
3867 case 0x86 ... 0x87: /* xchg */
3869 /* Write back the register source. */
3870 c->src.val = c->dst.val;
3871 write_register_operand(&c->src);
3873 * Write back the memory destination with implicit LOCK
3876 c->dst.val = c->src.orig_val;
3879 case 0x8c: /* mov r/m, sreg */
3880 if (c->modrm_reg > VCPU_SREG_GS) {
3881 rc = emulate_ud(ctxt);
3884 c->dst.val = ops->get_segment_selector(ctxt, c->modrm_reg);
3886 case 0x8d: /* lea r16/r32, m */
3887 c->dst.val = c->src.addr.mem.ea;
3889 case 0x8e: { /* mov seg, r/m16 */
3894 if (c->modrm_reg == VCPU_SREG_CS ||
3895 c->modrm_reg > VCPU_SREG_GS) {
3896 rc = emulate_ud(ctxt);
3900 if (c->modrm_reg == VCPU_SREG_SS)
3901 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3903 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
3905 c->dst.type = OP_NONE; /* Disable writeback. */
3908 case 0x8f: /* pop (sole member of Grp1a) */
3909 rc = emulate_grp1a(ctxt, ops);
3911 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3912 if (c->dst.addr.reg == &c->regs[VCPU_REGS_RAX])
3915 case 0x98: /* cbw/cwde/cdqe */
3916 switch (c->op_bytes) {
3917 case 2: c->dst.val = (s8)c->dst.val; break;
3918 case 4: c->dst.val = (s16)c->dst.val; break;
3919 case 8: c->dst.val = (s32)c->dst.val; break;
3922 case 0xa8 ... 0xa9: /* test ax, imm */
3927 case 0xc3: /* ret */
3928 c->dst.type = OP_REG;
3929 c->dst.addr.reg = &c->eip;
3930 c->dst.bytes = c->op_bytes;
3933 case 0xc4: /* les */
3934 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_ES);
3936 case 0xc5: /* lds */
3937 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_DS);
3939 case 0xcb: /* ret far */
3940 rc = emulate_ret_far(ctxt, ops);
3942 case 0xcc: /* int3 */
3945 case 0xcd: /* int n */
3948 rc = emulate_int(ctxt, ops, irq);
3950 case 0xce: /* into */
3951 if (ctxt->eflags & EFLG_OF) {
3956 case 0xcf: /* iret */
3957 rc = emulate_iret(ctxt, ops);
3959 case 0xd0 ... 0xd1: /* Grp2 */
3962 case 0xd2 ... 0xd3: /* Grp2 */
3963 c->src.val = c->regs[VCPU_REGS_RCX];
3966 case 0xe0 ... 0xe2: /* loop/loopz/loopnz */
3967 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3968 if (address_mask(c, c->regs[VCPU_REGS_RCX]) != 0 &&
3969 (c->b == 0xe2 || test_cc(c->b ^ 0x5, ctxt->eflags)))
3970 jmp_rel(c, c->src.val);
3972 case 0xe3: /* jcxz/jecxz/jrcxz */
3973 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0)
3974 jmp_rel(c, c->src.val);
3976 case 0xe4: /* inb */
3979 case 0xe6: /* outb */
3980 case 0xe7: /* out */
3982 case 0xe8: /* call (near) */ {
3983 long int rel = c->src.val;
3984 c->src.val = (unsigned long) c->eip;
3989 case 0xe9: /* jmp rel */
3991 case 0xea: { /* jmp far */
3994 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
3996 rc = load_segment_descriptor(ctxt, ops, sel, VCPU_SREG_CS);
3997 if (rc != X86EMUL_CONTINUE)
4001 memcpy(&c->eip, c->src.valptr, c->op_bytes);
4005 jmp: /* jmp rel short */
4006 jmp_rel(c, c->src.val);
4007 c->dst.type = OP_NONE; /* Disable writeback. */
4009 case 0xec: /* in al,dx */
4010 case 0xed: /* in (e/r)ax,dx */
4011 c->src.val = c->regs[VCPU_REGS_RDX];
4013 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
4015 goto done; /* IO is needed */
4017 case 0xee: /* out dx,al */
4018 case 0xef: /* out dx,(e/r)ax */
4019 c->dst.val = c->regs[VCPU_REGS_RDX];
4021 ops->pio_out_emulated(ctxt, c->src.bytes, c->dst.val,
4023 c->dst.type = OP_NONE; /* Disable writeback. */
4025 case 0xf4: /* hlt */
4026 ctxt->ops->halt(ctxt);
4028 case 0xf5: /* cmc */
4029 /* complement carry flag from eflags reg */
4030 ctxt->eflags ^= EFLG_CF;
4032 case 0xf6 ... 0xf7: /* Grp3 */
4033 rc = emulate_grp3(ctxt, ops);
4035 case 0xf8: /* clc */
4036 ctxt->eflags &= ~EFLG_CF;
4038 case 0xf9: /* stc */
4039 ctxt->eflags |= EFLG_CF;
4041 case 0xfa: /* cli */
4042 if (emulator_bad_iopl(ctxt, ops)) {
4043 rc = emulate_gp(ctxt, 0);
4046 ctxt->eflags &= ~X86_EFLAGS_IF;
4048 case 0xfb: /* sti */
4049 if (emulator_bad_iopl(ctxt, ops)) {
4050 rc = emulate_gp(ctxt, 0);
4053 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
4054 ctxt->eflags |= X86_EFLAGS_IF;
4057 case 0xfc: /* cld */
4058 ctxt->eflags &= ~EFLG_DF;
4060 case 0xfd: /* std */
4061 ctxt->eflags |= EFLG_DF;
4063 case 0xfe: /* Grp4 */
4065 rc = emulate_grp45(ctxt);
4067 case 0xff: /* Grp5 */
4068 if (c->modrm_reg == 5)
4072 goto cannot_emulate;
4075 if (rc != X86EMUL_CONTINUE)
4079 rc = writeback(ctxt, ops);
4080 if (rc != X86EMUL_CONTINUE)
4084 * restore dst type in case the decoding will be reused
4085 * (happens for string instruction )
4087 c->dst.type = saved_dst_type;
4089 if ((c->d & SrcMask) == SrcSI)
4090 string_addr_inc(ctxt, seg_override(ctxt, ops, c),
4091 VCPU_REGS_RSI, &c->src);
4093 if ((c->d & DstMask) == DstDI)
4094 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4097 if (c->rep_prefix && (c->d & String)) {
4098 struct read_cache *r = &ctxt->decode.io_read;
4099 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
4101 if (!string_insn_completed(ctxt)) {
4103 * Re-enter guest when pio read ahead buffer is empty
4104 * or, if it is not used, after each 1024 iteration.
4106 if ((r->end != 0 || c->regs[VCPU_REGS_RCX] & 0x3ff) &&
4107 (r->end == 0 || r->end != r->pos)) {
4109 * Reset read cache. Usually happens before
4110 * decode, but since instruction is restarted
4111 * we have to do it here.
4113 ctxt->decode.mem_read.end = 0;
4114 return EMULATION_RESTART;
4116 goto done; /* skip rip writeback */
4123 if (rc == X86EMUL_PROPAGATE_FAULT)
4124 ctxt->have_exception = true;
4125 if (rc == X86EMUL_INTERCEPTED)
4126 return EMULATION_INTERCEPTED;
4128 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4132 case 0x05: /* syscall */
4133 rc = emulate_syscall(ctxt, ops);
4138 case 0x09: /* wbinvd */
4139 (ctxt->ops->wbinvd)(ctxt);
4141 case 0x08: /* invd */
4142 case 0x0d: /* GrpP (prefetch) */
4143 case 0x18: /* Grp16 (prefetch/nop) */
4145 case 0x20: /* mov cr, reg */
4146 c->dst.val = ops->get_cr(ctxt, c->modrm_reg);
4148 case 0x21: /* mov from dr to reg */
4149 ops->get_dr(ctxt, c->modrm_reg, &c->dst.val);
4151 case 0x22: /* mov reg, cr */
4152 if (ops->set_cr(ctxt, c->modrm_reg, c->src.val)) {
4153 emulate_gp(ctxt, 0);
4154 rc = X86EMUL_PROPAGATE_FAULT;
4157 c->dst.type = OP_NONE;
4159 case 0x23: /* mov from reg to dr */
4160 if (ops->set_dr(ctxt, c->modrm_reg, c->src.val &
4161 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4162 ~0ULL : ~0U)) < 0) {
4163 /* #UD condition is already handled by the code above */
4164 emulate_gp(ctxt, 0);
4165 rc = X86EMUL_PROPAGATE_FAULT;
4169 c->dst.type = OP_NONE; /* no writeback */
4173 msr_data = (u32)c->regs[VCPU_REGS_RAX]
4174 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
4175 if (ops->set_msr(ctxt, c->regs[VCPU_REGS_RCX], msr_data)) {
4176 emulate_gp(ctxt, 0);
4177 rc = X86EMUL_PROPAGATE_FAULT;
4180 rc = X86EMUL_CONTINUE;
4184 if (ops->get_msr(ctxt, c->regs[VCPU_REGS_RCX], &msr_data)) {
4185 emulate_gp(ctxt, 0);
4186 rc = X86EMUL_PROPAGATE_FAULT;
4189 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
4190 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
4192 rc = X86EMUL_CONTINUE;
4194 case 0x34: /* sysenter */
4195 rc = emulate_sysenter(ctxt, ops);
4197 case 0x35: /* sysexit */
4198 rc = emulate_sysexit(ctxt, ops);
4200 case 0x40 ... 0x4f: /* cmov */
4201 c->dst.val = c->dst.orig_val = c->src.val;
4202 if (!test_cc(c->b, ctxt->eflags))
4203 c->dst.type = OP_NONE; /* no writeback */
4205 case 0x80 ... 0x8f: /* jnz rel, etc*/
4206 if (test_cc(c->b, ctxt->eflags))
4207 jmp_rel(c, c->src.val);
4209 case 0x90 ... 0x9f: /* setcc r/m8 */
4210 c->dst.val = test_cc(c->b, ctxt->eflags);
4212 case 0xa0: /* push fs */
4213 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
4215 case 0xa1: /* pop fs */
4216 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
4220 c->dst.type = OP_NONE;
4221 /* only subword offset */
4222 c->src.val &= (c->dst.bytes << 3) - 1;
4223 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
4225 case 0xa4: /* shld imm8, r, r/m */
4226 case 0xa5: /* shld cl, r, r/m */
4227 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
4229 case 0xa8: /* push gs */
4230 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
4232 case 0xa9: /* pop gs */
4233 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
4237 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
4239 case 0xac: /* shrd imm8, r, r/m */
4240 case 0xad: /* shrd cl, r, r/m */
4241 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
4243 case 0xae: /* clflush */
4245 case 0xb0 ... 0xb1: /* cmpxchg */
4247 * Save real source value, then compare EAX against
4250 c->src.orig_val = c->src.val;
4251 c->src.val = c->regs[VCPU_REGS_RAX];
4252 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
4253 if (ctxt->eflags & EFLG_ZF) {
4254 /* Success: write back to memory. */
4255 c->dst.val = c->src.orig_val;
4257 /* Failure: write the value we saw to EAX. */
4258 c->dst.type = OP_REG;
4259 c->dst.addr.reg = (unsigned long *)&c->regs[VCPU_REGS_RAX];
4262 case 0xb2: /* lss */
4263 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_SS);
4267 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
4269 case 0xb4: /* lfs */
4270 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_FS);
4272 case 0xb5: /* lgs */
4273 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_GS);
4275 case 0xb6 ... 0xb7: /* movzx */
4276 c->dst.bytes = c->op_bytes;
4277 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
4280 case 0xba: /* Grp8 */
4281 switch (c->modrm_reg & 3) {
4294 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
4296 case 0xbc: { /* bsf */
4298 __asm__ ("bsf %2, %0; setz %1"
4299 : "=r"(c->dst.val), "=q"(zf)
4301 ctxt->eflags &= ~X86_EFLAGS_ZF;
4303 ctxt->eflags |= X86_EFLAGS_ZF;
4304 c->dst.type = OP_NONE; /* Disable writeback. */
4308 case 0xbd: { /* bsr */
4310 __asm__ ("bsr %2, %0; setz %1"
4311 : "=r"(c->dst.val), "=q"(zf)
4313 ctxt->eflags &= ~X86_EFLAGS_ZF;
4315 ctxt->eflags |= X86_EFLAGS_ZF;
4316 c->dst.type = OP_NONE; /* Disable writeback. */
4320 case 0xbe ... 0xbf: /* movsx */
4321 c->dst.bytes = c->op_bytes;
4322 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
4325 case 0xc0 ... 0xc1: /* xadd */
4326 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
4327 /* Write back the register source. */
4328 c->src.val = c->dst.orig_val;
4329 write_register_operand(&c->src);
4331 case 0xc3: /* movnti */
4332 c->dst.bytes = c->op_bytes;
4333 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
4336 case 0xc7: /* Grp9 (cmpxchg8b) */
4337 rc = emulate_grp9(ctxt, ops);
4340 goto cannot_emulate;
4343 if (rc != X86EMUL_CONTINUE)
4349 return EMULATION_FAILED;