1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
35 #define OpImplicit 1ull /* No generic decode */
36 #define OpReg 2ull /* Register */
37 #define OpMem 3ull /* Memory */
38 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
39 #define OpDI 5ull /* ES:DI/EDI/RDI */
40 #define OpMem64 6ull /* Memory, 64-bit */
41 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
42 #define OpDX 8ull /* DX register */
43 #define OpCL 9ull /* CL register (for shifts) */
44 #define OpImmByte 10ull /* 8-bit sign extended immediate */
45 #define OpOne 11ull /* Implied 1 */
46 #define OpImm 12ull /* Sign extended immediate */
47 #define OpMem16 13ull /* Memory operand (16-bit). */
48 #define OpMem32 14ull /* Memory operand (32-bit). */
49 #define OpImmU 15ull /* Immediate operand, zero extended */
50 #define OpSI 16ull /* SI/ESI/RSI */
51 #define OpImmFAddr 17ull /* Immediate far address */
52 #define OpMemFAddr 18ull /* Far address in memory */
53 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
54 #define OpES 20ull /* ES */
55 #define OpCS 21ull /* CS */
56 #define OpSS 22ull /* SS */
57 #define OpDS 23ull /* DS */
58 #define OpFS 24ull /* FS */
59 #define OpGS 25ull /* GS */
61 #define OpBits 5 /* Width of operand field */
62 #define OpMask ((1ull << OpBits) - 1)
65 * Opcode effective-address decode tables.
66 * Note that we only emulate instructions that have at least one memory
67 * operand (excluding implicit stack references). We assume that stack
68 * references and instruction fetches will never occur in special memory
69 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
73 /* Operand sizes: 8-bit operands or specified/overridden size. */
74 #define ByteOp (1<<0) /* 8-bit operands. */
75 /* Destination operand type. */
77 #define ImplicitOps (OpImplicit << DstShift)
78 #define DstReg (OpReg << DstShift)
79 #define DstMem (OpMem << DstShift)
80 #define DstAcc (OpAcc << DstShift)
81 #define DstDI (OpDI << DstShift)
82 #define DstMem64 (OpMem64 << DstShift)
83 #define DstImmUByte (OpImmUByte << DstShift)
84 #define DstDX (OpDX << DstShift)
85 #define DstMask (OpMask << DstShift)
86 /* Source operand type. */
88 #define SrcNone (OpNone << SrcShift)
89 #define SrcReg (OpReg << SrcShift)
90 #define SrcMem (OpMem << SrcShift)
91 #define SrcMem16 (OpMem16 << SrcShift)
92 #define SrcMem32 (OpMem32 << SrcShift)
93 #define SrcImm (OpImm << SrcShift)
94 #define SrcImmByte (OpImmByte << SrcShift)
95 #define SrcOne (OpOne << SrcShift)
96 #define SrcImmUByte (OpImmUByte << SrcShift)
97 #define SrcImmU (OpImmU << SrcShift)
98 #define SrcSI (OpSI << SrcShift)
99 #define SrcImmFAddr (OpImmFAddr << SrcShift)
100 #define SrcMemFAddr (OpMemFAddr << SrcShift)
101 #define SrcAcc (OpAcc << SrcShift)
102 #define SrcImmU16 (OpImmU16 << SrcShift)
103 #define SrcDX (OpDX << SrcShift)
104 #define SrcMask (OpMask << SrcShift)
105 #define BitOp (1<<11)
106 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
107 #define String (1<<13) /* String instruction (rep capable) */
108 #define Stack (1<<14) /* Stack instruction (push/pop) */
109 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
110 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
111 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
112 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
113 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
114 #define Sse (1<<18) /* SSE Vector instruction */
115 /* Generic ModRM decode. */
116 #define ModRM (1<<19)
117 /* Destination is only written; never read. */
120 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
121 #define VendorSpecific (1<<22) /* Vendor specific instruction */
122 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
123 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
124 #define Undefined (1<<25) /* No Such Instruction */
125 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
126 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
128 /* Source 2 operand type */
129 #define Src2Shift (29)
130 #define Src2None (OpNone << Src2Shift)
131 #define Src2CL (OpCL << Src2Shift)
132 #define Src2ImmByte (OpImmByte << Src2Shift)
133 #define Src2One (OpOne << Src2Shift)
134 #define Src2Imm (OpImm << Src2Shift)
135 #define Src2ES (OpES << Src2Shift)
136 #define Src2CS (OpCS << Src2Shift)
137 #define Src2SS (OpSS << Src2Shift)
138 #define Src2DS (OpDS << Src2Shift)
139 #define Src2FS (OpFS << Src2Shift)
140 #define Src2GS (OpGS << Src2Shift)
141 #define Src2Mask (OpMask << Src2Shift)
143 #define X2(x...) x, x
144 #define X3(x...) X2(x), x
145 #define X4(x...) X2(x), X2(x)
146 #define X5(x...) X4(x), x
147 #define X6(x...) X4(x), X2(x)
148 #define X7(x...) X4(x), X3(x)
149 #define X8(x...) X4(x), X4(x)
150 #define X16(x...) X8(x), X8(x)
156 int (*execute)(struct x86_emulate_ctxt *ctxt);
157 struct opcode *group;
158 struct group_dual *gdual;
159 struct gprefix *gprefix;
161 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
165 struct opcode mod012[8];
166 struct opcode mod3[8];
170 struct opcode pfx_no;
171 struct opcode pfx_66;
172 struct opcode pfx_f2;
173 struct opcode pfx_f3;
176 /* EFLAGS bit definitions. */
177 #define EFLG_ID (1<<21)
178 #define EFLG_VIP (1<<20)
179 #define EFLG_VIF (1<<19)
180 #define EFLG_AC (1<<18)
181 #define EFLG_VM (1<<17)
182 #define EFLG_RF (1<<16)
183 #define EFLG_IOPL (3<<12)
184 #define EFLG_NT (1<<14)
185 #define EFLG_OF (1<<11)
186 #define EFLG_DF (1<<10)
187 #define EFLG_IF (1<<9)
188 #define EFLG_TF (1<<8)
189 #define EFLG_SF (1<<7)
190 #define EFLG_ZF (1<<6)
191 #define EFLG_AF (1<<4)
192 #define EFLG_PF (1<<2)
193 #define EFLG_CF (1<<0)
195 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
196 #define EFLG_RESERVED_ONE_MASK 2
199 * Instruction emulation:
200 * Most instructions are emulated directly via a fragment of inline assembly
201 * code. This allows us to save/restore EFLAGS and thus very easily pick up
202 * any modified flags.
205 #if defined(CONFIG_X86_64)
206 #define _LO32 "k" /* force 32-bit operand */
207 #define _STK "%%rsp" /* stack pointer */
208 #elif defined(__i386__)
209 #define _LO32 "" /* force 32-bit operand */
210 #define _STK "%%esp" /* stack pointer */
214 * These EFLAGS bits are restored from saved value during emulation, and
215 * any changes are written back to the saved value after emulation.
217 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
219 /* Before executing instruction: restore necessary bits in EFLAGS. */
220 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
221 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
222 "movl %"_sav",%"_LO32 _tmp"; " \
225 "movl %"_msk",%"_LO32 _tmp"; " \
226 "andl %"_LO32 _tmp",("_STK"); " \
228 "notl %"_LO32 _tmp"; " \
229 "andl %"_LO32 _tmp",("_STK"); " \
230 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
232 "orl %"_LO32 _tmp",("_STK"); " \
236 /* After executing instruction: write-back necessary bits in EFLAGS. */
237 #define _POST_EFLAGS(_sav, _msk, _tmp) \
238 /* _sav |= EFLAGS & _msk; */ \
241 "andl %"_msk",%"_LO32 _tmp"; " \
242 "orl %"_LO32 _tmp",%"_sav"; "
250 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
252 __asm__ __volatile__ ( \
253 _PRE_EFLAGS("0", "4", "2") \
254 _op _suffix " %"_x"3,%1; " \
255 _POST_EFLAGS("0", "4", "2") \
256 : "=m" ((ctxt)->eflags), \
257 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
259 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
263 /* Raw emulation: instruction has two explicit operands. */
264 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
266 unsigned long _tmp; \
268 switch ((ctxt)->dst.bytes) { \
270 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
273 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
276 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
281 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
283 unsigned long _tmp; \
284 switch ((ctxt)->dst.bytes) { \
286 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
289 __emulate_2op_nobyte(ctxt, _op, \
290 _wx, _wy, _lx, _ly, _qx, _qy); \
295 /* Source operand is byte-sized and may be restricted to just %cl. */
296 #define emulate_2op_SrcB(ctxt, _op) \
297 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
299 /* Source operand is byte, word, long or quad sized. */
300 #define emulate_2op_SrcV(ctxt, _op) \
301 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
303 /* Source operand is word, long or quad sized. */
304 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
305 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
307 /* Instruction has three operands and one operand is stored in ECX register */
308 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
310 unsigned long _tmp; \
311 _type _clv = (ctxt)->src2.val; \
312 _type _srcv = (ctxt)->src.val; \
313 _type _dstv = (ctxt)->dst.val; \
315 __asm__ __volatile__ ( \
316 _PRE_EFLAGS("0", "5", "2") \
317 _op _suffix " %4,%1 \n" \
318 _POST_EFLAGS("0", "5", "2") \
319 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
320 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
323 (ctxt)->src2.val = (unsigned long) _clv; \
324 (ctxt)->src2.val = (unsigned long) _srcv; \
325 (ctxt)->dst.val = (unsigned long) _dstv; \
328 #define emulate_2op_cl(ctxt, _op) \
330 switch ((ctxt)->dst.bytes) { \
332 __emulate_2op_cl(ctxt, _op, "w", u16); \
335 __emulate_2op_cl(ctxt, _op, "l", u32); \
338 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
343 #define __emulate_1op(ctxt, _op, _suffix) \
345 unsigned long _tmp; \
347 __asm__ __volatile__ ( \
348 _PRE_EFLAGS("0", "3", "2") \
349 _op _suffix " %1; " \
350 _POST_EFLAGS("0", "3", "2") \
351 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
353 : "i" (EFLAGS_MASK)); \
356 /* Instruction has only one explicit operand (no source operand). */
357 #define emulate_1op(ctxt, _op) \
359 switch ((ctxt)->dst.bytes) { \
360 case 1: __emulate_1op(ctxt, _op, "b"); break; \
361 case 2: __emulate_1op(ctxt, _op, "w"); break; \
362 case 4: __emulate_1op(ctxt, _op, "l"); break; \
363 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
367 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
369 unsigned long _tmp; \
370 ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX]; \
371 ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX]; \
373 __asm__ __volatile__ ( \
374 _PRE_EFLAGS("0", "5", "1") \
376 _op _suffix " %6; " \
378 _POST_EFLAGS("0", "5", "1") \
379 ".pushsection .fixup,\"ax\" \n\t" \
380 "3: movb $1, %4 \n\t" \
383 _ASM_EXTABLE(1b, 3b) \
384 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
385 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
386 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val), \
387 "a" (*rax), "d" (*rdx)); \
390 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
391 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
393 switch((ctxt)->src.bytes) { \
395 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
398 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
401 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
404 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
409 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
410 enum x86_intercept intercept,
411 enum x86_intercept_stage stage)
413 struct x86_instruction_info info = {
414 .intercept = intercept,
415 .rep_prefix = ctxt->rep_prefix,
416 .modrm_mod = ctxt->modrm_mod,
417 .modrm_reg = ctxt->modrm_reg,
418 .modrm_rm = ctxt->modrm_rm,
419 .src_val = ctxt->src.val64,
420 .src_bytes = ctxt->src.bytes,
421 .dst_bytes = ctxt->dst.bytes,
422 .ad_bytes = ctxt->ad_bytes,
423 .next_rip = ctxt->eip,
426 return ctxt->ops->intercept(ctxt, &info, stage);
429 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
431 return (1UL << (ctxt->ad_bytes << 3)) - 1;
434 /* Access/update address held in a register, based on addressing mode. */
435 static inline unsigned long
436 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
438 if (ctxt->ad_bytes == sizeof(unsigned long))
441 return reg & ad_mask(ctxt);
444 static inline unsigned long
445 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
447 return address_mask(ctxt, reg);
451 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
453 if (ctxt->ad_bytes == sizeof(unsigned long))
456 *reg = (*reg & ~ad_mask(ctxt)) | ((*reg + inc) & ad_mask(ctxt));
459 static u32 desc_limit_scaled(struct desc_struct *desc)
461 u32 limit = get_desc_limit(desc);
463 return desc->g ? (limit << 12) | 0xfff : limit;
466 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
468 ctxt->has_seg_override = true;
469 ctxt->seg_override = seg;
472 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
474 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
477 return ctxt->ops->get_cached_segment_base(ctxt, seg);
480 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
482 if (!ctxt->has_seg_override)
485 return ctxt->seg_override;
488 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
489 u32 error, bool valid)
491 ctxt->exception.vector = vec;
492 ctxt->exception.error_code = error;
493 ctxt->exception.error_code_valid = valid;
494 return X86EMUL_PROPAGATE_FAULT;
497 static int emulate_db(struct x86_emulate_ctxt *ctxt)
499 return emulate_exception(ctxt, DB_VECTOR, 0, false);
502 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
504 return emulate_exception(ctxt, GP_VECTOR, err, true);
507 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
509 return emulate_exception(ctxt, SS_VECTOR, err, true);
512 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
514 return emulate_exception(ctxt, UD_VECTOR, 0, false);
517 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
519 return emulate_exception(ctxt, TS_VECTOR, err, true);
522 static int emulate_de(struct x86_emulate_ctxt *ctxt)
524 return emulate_exception(ctxt, DE_VECTOR, 0, false);
527 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
529 return emulate_exception(ctxt, NM_VECTOR, 0, false);
532 static inline void assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
534 switch (ctxt->op_bytes) {
536 ctxt->_eip = (u16)dst;
539 ctxt->_eip = (u32)dst;
545 WARN(1, "unsupported eip assignment size\n");
549 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
551 assign_eip_near(ctxt, ctxt->_eip + rel);
554 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
557 struct desc_struct desc;
559 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
563 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
568 struct desc_struct desc;
570 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
571 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
574 static int __linearize(struct x86_emulate_ctxt *ctxt,
575 struct segmented_address addr,
576 unsigned size, bool write, bool fetch,
579 struct desc_struct desc;
586 la = seg_base(ctxt, addr.seg) + addr.ea;
587 switch (ctxt->mode) {
588 case X86EMUL_MODE_REAL:
590 case X86EMUL_MODE_PROT64:
591 if (((signed long)la << 16) >> 16 != la)
592 return emulate_gp(ctxt, 0);
595 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
599 /* code segment or read-only data segment */
600 if (((desc.type & 8) || !(desc.type & 2)) && write)
602 /* unreadable code segment */
603 if (!fetch && (desc.type & 8) && !(desc.type & 2))
605 lim = desc_limit_scaled(&desc);
606 if ((desc.type & 8) || !(desc.type & 4)) {
607 /* expand-up segment */
608 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
611 /* exapand-down segment */
612 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
614 lim = desc.d ? 0xffffffff : 0xffff;
615 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
618 cpl = ctxt->ops->cpl(ctxt);
621 if (!(desc.type & 8)) {
625 } else if ((desc.type & 8) && !(desc.type & 4)) {
626 /* nonconforming code segment */
629 } else if ((desc.type & 8) && (desc.type & 4)) {
630 /* conforming code segment */
636 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
639 return X86EMUL_CONTINUE;
641 if (addr.seg == VCPU_SREG_SS)
642 return emulate_ss(ctxt, addr.seg);
644 return emulate_gp(ctxt, addr.seg);
647 static int linearize(struct x86_emulate_ctxt *ctxt,
648 struct segmented_address addr,
649 unsigned size, bool write,
652 return __linearize(ctxt, addr, size, write, false, linear);
656 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
657 struct segmented_address addr,
664 rc = linearize(ctxt, addr, size, false, &linear);
665 if (rc != X86EMUL_CONTINUE)
667 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
671 * Fetch the next byte of the instruction being emulated which is pointed to
672 * by ctxt->_eip, then increment ctxt->_eip.
674 * Also prefetch the remaining bytes of the instruction without crossing page
675 * boundary if they are not in fetch_cache yet.
677 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
679 struct fetch_cache *fc = &ctxt->fetch;
683 if (ctxt->_eip == fc->end) {
684 unsigned long linear;
685 struct segmented_address addr = { .seg = VCPU_SREG_CS,
687 cur_size = fc->end - fc->start;
688 size = min(15UL - cur_size,
689 PAGE_SIZE - offset_in_page(ctxt->_eip));
690 rc = __linearize(ctxt, addr, size, false, true, &linear);
691 if (unlikely(rc != X86EMUL_CONTINUE))
693 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
694 size, &ctxt->exception);
695 if (unlikely(rc != X86EMUL_CONTINUE))
699 *dest = fc->data[ctxt->_eip - fc->start];
701 return X86EMUL_CONTINUE;
704 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
705 void *dest, unsigned size)
709 /* x86 instructions are limited to 15 bytes. */
710 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
711 return X86EMUL_UNHANDLEABLE;
713 rc = do_insn_fetch_byte(ctxt, dest++);
714 if (rc != X86EMUL_CONTINUE)
717 return X86EMUL_CONTINUE;
720 /* Fetch next part of the instruction being emulated. */
721 #define insn_fetch(_type, _ctxt) \
722 ({ unsigned long _x; \
723 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
724 if (rc != X86EMUL_CONTINUE) \
729 #define insn_fetch_arr(_arr, _size, _ctxt) \
730 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
731 if (rc != X86EMUL_CONTINUE) \
736 * Given the 'reg' portion of a ModRM byte, and a register block, return a
737 * pointer into the block that addresses the relevant register.
738 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
740 static void *decode_register(u8 modrm_reg, unsigned long *regs,
745 p = ®s[modrm_reg];
746 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
747 p = (unsigned char *)®s[modrm_reg & 3] + 1;
751 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
752 struct segmented_address addr,
753 u16 *size, unsigned long *address, int op_bytes)
760 rc = segmented_read_std(ctxt, addr, size, 2);
761 if (rc != X86EMUL_CONTINUE)
764 rc = segmented_read_std(ctxt, addr, address, op_bytes);
768 static int test_cc(unsigned int condition, unsigned int flags)
772 switch ((condition & 15) >> 1) {
774 rc |= (flags & EFLG_OF);
776 case 1: /* b/c/nae */
777 rc |= (flags & EFLG_CF);
780 rc |= (flags & EFLG_ZF);
783 rc |= (flags & (EFLG_CF|EFLG_ZF));
786 rc |= (flags & EFLG_SF);
789 rc |= (flags & EFLG_PF);
792 rc |= (flags & EFLG_ZF);
795 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
799 /* Odd condition identifiers (lsb == 1) have inverted sense. */
800 return (!!rc ^ (condition & 1));
803 static void fetch_register_operand(struct operand *op)
807 op->val = *(u8 *)op->addr.reg;
810 op->val = *(u16 *)op->addr.reg;
813 op->val = *(u32 *)op->addr.reg;
816 op->val = *(u64 *)op->addr.reg;
821 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
823 ctxt->ops->get_fpu(ctxt);
825 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
826 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
827 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
828 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
829 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
830 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
831 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
832 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
834 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
835 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
836 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
837 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
838 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
839 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
840 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
841 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
845 ctxt->ops->put_fpu(ctxt);
848 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
851 ctxt->ops->get_fpu(ctxt);
853 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
854 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
855 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
856 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
857 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
858 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
859 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
860 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
862 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
863 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
864 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
865 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
866 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
867 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
868 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
869 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
873 ctxt->ops->put_fpu(ctxt);
876 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
880 unsigned reg = ctxt->modrm_reg;
881 int highbyte_regs = ctxt->rex_prefix == 0;
883 if (!(ctxt->d & ModRM))
884 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
890 read_sse_reg(ctxt, &op->vec_val, reg);
895 if ((ctxt->d & ByteOp) && !inhibit_bytereg) {
896 op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
899 op->addr.reg = decode_register(reg, ctxt->regs, 0);
900 op->bytes = ctxt->op_bytes;
902 fetch_register_operand(op);
903 op->orig_val = op->val;
906 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
910 int index_reg = 0, base_reg = 0, scale;
911 int rc = X86EMUL_CONTINUE;
914 if (ctxt->rex_prefix) {
915 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
916 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
917 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
920 ctxt->modrm = insn_fetch(u8, ctxt);
921 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
922 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
923 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
924 ctxt->modrm_seg = VCPU_SREG_DS;
926 if (ctxt->modrm_mod == 3) {
928 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
929 op->addr.reg = decode_register(ctxt->modrm_rm,
930 ctxt->regs, ctxt->d & ByteOp);
934 op->addr.xmm = ctxt->modrm_rm;
935 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
938 fetch_register_operand(op);
944 if (ctxt->ad_bytes == 2) {
945 unsigned bx = ctxt->regs[VCPU_REGS_RBX];
946 unsigned bp = ctxt->regs[VCPU_REGS_RBP];
947 unsigned si = ctxt->regs[VCPU_REGS_RSI];
948 unsigned di = ctxt->regs[VCPU_REGS_RDI];
950 /* 16-bit ModR/M decode. */
951 switch (ctxt->modrm_mod) {
953 if (ctxt->modrm_rm == 6)
954 modrm_ea += insn_fetch(u16, ctxt);
957 modrm_ea += insn_fetch(s8, ctxt);
960 modrm_ea += insn_fetch(u16, ctxt);
963 switch (ctxt->modrm_rm) {
983 if (ctxt->modrm_mod != 0)
990 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
991 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
992 ctxt->modrm_seg = VCPU_SREG_SS;
993 modrm_ea = (u16)modrm_ea;
995 /* 32/64-bit ModR/M decode. */
996 if ((ctxt->modrm_rm & 7) == 4) {
997 sib = insn_fetch(u8, ctxt);
998 index_reg |= (sib >> 3) & 7;
1002 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1003 modrm_ea += insn_fetch(s32, ctxt);
1005 modrm_ea += ctxt->regs[base_reg];
1007 modrm_ea += ctxt->regs[index_reg] << scale;
1008 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1009 if (ctxt->mode == X86EMUL_MODE_PROT64)
1010 ctxt->rip_relative = 1;
1012 modrm_ea += ctxt->regs[ctxt->modrm_rm];
1013 switch (ctxt->modrm_mod) {
1015 if (ctxt->modrm_rm == 5)
1016 modrm_ea += insn_fetch(s32, ctxt);
1019 modrm_ea += insn_fetch(s8, ctxt);
1022 modrm_ea += insn_fetch(s32, ctxt);
1026 op->addr.mem.ea = modrm_ea;
1031 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1034 int rc = X86EMUL_CONTINUE;
1037 switch (ctxt->ad_bytes) {
1039 op->addr.mem.ea = insn_fetch(u16, ctxt);
1042 op->addr.mem.ea = insn_fetch(u32, ctxt);
1045 op->addr.mem.ea = insn_fetch(u64, ctxt);
1052 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1056 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1057 mask = ~(ctxt->dst.bytes * 8 - 1);
1059 if (ctxt->src.bytes == 2)
1060 sv = (s16)ctxt->src.val & (s16)mask;
1061 else if (ctxt->src.bytes == 4)
1062 sv = (s32)ctxt->src.val & (s32)mask;
1064 ctxt->dst.addr.mem.ea += (sv >> 3);
1067 /* only subword offset */
1068 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1071 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1072 unsigned long addr, void *dest, unsigned size)
1075 struct read_cache *mc = &ctxt->mem_read;
1078 int n = min(size, 8u);
1080 if (mc->pos < mc->end)
1083 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1085 if (rc != X86EMUL_CONTINUE)
1090 memcpy(dest, mc->data + mc->pos, n);
1095 return X86EMUL_CONTINUE;
1098 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1099 struct segmented_address addr,
1106 rc = linearize(ctxt, addr, size, false, &linear);
1107 if (rc != X86EMUL_CONTINUE)
1109 return read_emulated(ctxt, linear, data, size);
1112 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1113 struct segmented_address addr,
1120 rc = linearize(ctxt, addr, size, true, &linear);
1121 if (rc != X86EMUL_CONTINUE)
1123 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1127 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1128 struct segmented_address addr,
1129 const void *orig_data, const void *data,
1135 rc = linearize(ctxt, addr, size, true, &linear);
1136 if (rc != X86EMUL_CONTINUE)
1138 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1139 size, &ctxt->exception);
1142 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1143 unsigned int size, unsigned short port,
1146 struct read_cache *rc = &ctxt->io_read;
1148 if (rc->pos == rc->end) { /* refill pio read ahead */
1149 unsigned int in_page, n;
1150 unsigned int count = ctxt->rep_prefix ?
1151 address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
1152 in_page = (ctxt->eflags & EFLG_DF) ?
1153 offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
1154 PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
1155 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1159 rc->pos = rc->end = 0;
1160 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1165 memcpy(dest, rc->data + rc->pos, size);
1170 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1171 u16 selector, struct desc_ptr *dt)
1173 struct x86_emulate_ops *ops = ctxt->ops;
1175 if (selector & 1 << 2) {
1176 struct desc_struct desc;
1179 memset (dt, 0, sizeof *dt);
1180 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1183 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1184 dt->address = get_desc_base(&desc);
1186 ops->get_gdt(ctxt, dt);
1189 /* allowed just for 8 bytes segments */
1190 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1191 u16 selector, struct desc_struct *desc)
1194 u16 index = selector >> 3;
1197 get_descriptor_table_ptr(ctxt, selector, &dt);
1199 if (dt.size < index * 8 + 7)
1200 return emulate_gp(ctxt, selector & 0xfffc);
1202 addr = dt.address + index * 8;
1203 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1207 /* allowed just for 8 bytes segments */
1208 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1209 u16 selector, struct desc_struct *desc)
1212 u16 index = selector >> 3;
1215 get_descriptor_table_ptr(ctxt, selector, &dt);
1217 if (dt.size < index * 8 + 7)
1218 return emulate_gp(ctxt, selector & 0xfffc);
1220 addr = dt.address + index * 8;
1221 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1225 /* Does not support long mode */
1226 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1227 u16 selector, int seg)
1229 struct desc_struct seg_desc;
1231 unsigned err_vec = GP_VECTOR;
1233 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1236 memset(&seg_desc, 0, sizeof seg_desc);
1238 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1239 || ctxt->mode == X86EMUL_MODE_REAL) {
1240 /* set real mode segment descriptor */
1241 set_desc_base(&seg_desc, selector << 4);
1242 set_desc_limit(&seg_desc, 0xffff);
1249 /* NULL selector is not valid for TR, CS and SS */
1250 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1254 /* TR should be in GDT only */
1255 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1258 if (null_selector) /* for NULL selector skip all following checks */
1261 ret = read_segment_descriptor(ctxt, selector, &seg_desc);
1262 if (ret != X86EMUL_CONTINUE)
1265 err_code = selector & 0xfffc;
1266 err_vec = GP_VECTOR;
1268 /* can't load system descriptor into segment selecor */
1269 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1273 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1279 cpl = ctxt->ops->cpl(ctxt);
1284 * segment is not a writable data segment or segment
1285 * selector's RPL != CPL or segment selector's RPL != CPL
1287 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1291 if (!(seg_desc.type & 8))
1294 if (seg_desc.type & 4) {
1300 if (rpl > cpl || dpl != cpl)
1303 /* CS(RPL) <- CPL */
1304 selector = (selector & 0xfffc) | cpl;
1307 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1310 case VCPU_SREG_LDTR:
1311 if (seg_desc.s || seg_desc.type != 2)
1314 default: /* DS, ES, FS, or GS */
1316 * segment is not a data or readable code segment or
1317 * ((segment is a data or nonconforming code segment)
1318 * and (both RPL and CPL > DPL))
1320 if ((seg_desc.type & 0xa) == 0x8 ||
1321 (((seg_desc.type & 0xc) != 0xc) &&
1322 (rpl > dpl && cpl > dpl)))
1328 /* mark segment as accessed */
1330 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1331 if (ret != X86EMUL_CONTINUE)
1335 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1336 return X86EMUL_CONTINUE;
1338 emulate_exception(ctxt, err_vec, err_code, true);
1339 return X86EMUL_PROPAGATE_FAULT;
1342 static void write_register_operand(struct operand *op)
1344 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1345 switch (op->bytes) {
1347 *(u8 *)op->addr.reg = (u8)op->val;
1350 *(u16 *)op->addr.reg = (u16)op->val;
1353 *op->addr.reg = (u32)op->val;
1354 break; /* 64b: zero-extend */
1356 *op->addr.reg = op->val;
1361 static int writeback(struct x86_emulate_ctxt *ctxt)
1365 switch (ctxt->dst.type) {
1367 write_register_operand(&ctxt->dst);
1370 if (ctxt->lock_prefix)
1371 rc = segmented_cmpxchg(ctxt,
1373 &ctxt->dst.orig_val,
1377 rc = segmented_write(ctxt,
1381 if (rc != X86EMUL_CONTINUE)
1385 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1393 return X86EMUL_CONTINUE;
1396 static int em_push(struct x86_emulate_ctxt *ctxt)
1398 struct segmented_address addr;
1400 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], -ctxt->op_bytes);
1401 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1402 addr.seg = VCPU_SREG_SS;
1404 /* Disable writeback. */
1405 ctxt->dst.type = OP_NONE;
1406 return segmented_write(ctxt, addr, &ctxt->src.val, ctxt->op_bytes);
1409 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1410 void *dest, int len)
1413 struct segmented_address addr;
1415 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1416 addr.seg = VCPU_SREG_SS;
1417 rc = segmented_read(ctxt, addr, dest, len);
1418 if (rc != X86EMUL_CONTINUE)
1421 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], len);
1425 static int em_pop(struct x86_emulate_ctxt *ctxt)
1427 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1430 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1431 void *dest, int len)
1434 unsigned long val, change_mask;
1435 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1436 int cpl = ctxt->ops->cpl(ctxt);
1438 rc = emulate_pop(ctxt, &val, len);
1439 if (rc != X86EMUL_CONTINUE)
1442 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1443 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1445 switch(ctxt->mode) {
1446 case X86EMUL_MODE_PROT64:
1447 case X86EMUL_MODE_PROT32:
1448 case X86EMUL_MODE_PROT16:
1450 change_mask |= EFLG_IOPL;
1452 change_mask |= EFLG_IF;
1454 case X86EMUL_MODE_VM86:
1456 return emulate_gp(ctxt, 0);
1457 change_mask |= EFLG_IF;
1459 default: /* real mode */
1460 change_mask |= (EFLG_IOPL | EFLG_IF);
1464 *(unsigned long *)dest =
1465 (ctxt->eflags & ~change_mask) | (val & change_mask);
1470 static int em_popf(struct x86_emulate_ctxt *ctxt)
1472 ctxt->dst.type = OP_REG;
1473 ctxt->dst.addr.reg = &ctxt->eflags;
1474 ctxt->dst.bytes = ctxt->op_bytes;
1475 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1478 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1480 int seg = ctxt->src2.val;
1482 ctxt->src.val = get_segment_selector(ctxt, seg);
1484 return em_push(ctxt);
1487 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1489 int seg = ctxt->src2.val;
1490 unsigned long selector;
1493 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1494 if (rc != X86EMUL_CONTINUE)
1497 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1501 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1503 unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
1504 int rc = X86EMUL_CONTINUE;
1505 int reg = VCPU_REGS_RAX;
1507 while (reg <= VCPU_REGS_RDI) {
1508 (reg == VCPU_REGS_RSP) ?
1509 (ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
1512 if (rc != X86EMUL_CONTINUE)
1521 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1523 ctxt->src.val = (unsigned long)ctxt->eflags;
1524 return em_push(ctxt);
1527 static int em_popa(struct x86_emulate_ctxt *ctxt)
1529 int rc = X86EMUL_CONTINUE;
1530 int reg = VCPU_REGS_RDI;
1532 while (reg >= VCPU_REGS_RAX) {
1533 if (reg == VCPU_REGS_RSP) {
1534 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP],
1539 rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
1540 if (rc != X86EMUL_CONTINUE)
1547 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1549 struct x86_emulate_ops *ops = ctxt->ops;
1556 /* TODO: Add limit checks */
1557 ctxt->src.val = ctxt->eflags;
1559 if (rc != X86EMUL_CONTINUE)
1562 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1564 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1566 if (rc != X86EMUL_CONTINUE)
1569 ctxt->src.val = ctxt->_eip;
1571 if (rc != X86EMUL_CONTINUE)
1574 ops->get_idt(ctxt, &dt);
1576 eip_addr = dt.address + (irq << 2);
1577 cs_addr = dt.address + (irq << 2) + 2;
1579 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1580 if (rc != X86EMUL_CONTINUE)
1583 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1584 if (rc != X86EMUL_CONTINUE)
1587 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1588 if (rc != X86EMUL_CONTINUE)
1596 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1598 switch(ctxt->mode) {
1599 case X86EMUL_MODE_REAL:
1600 return emulate_int_real(ctxt, irq);
1601 case X86EMUL_MODE_VM86:
1602 case X86EMUL_MODE_PROT16:
1603 case X86EMUL_MODE_PROT32:
1604 case X86EMUL_MODE_PROT64:
1606 /* Protected mode interrupts unimplemented yet */
1607 return X86EMUL_UNHANDLEABLE;
1611 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1613 int rc = X86EMUL_CONTINUE;
1614 unsigned long temp_eip = 0;
1615 unsigned long temp_eflags = 0;
1616 unsigned long cs = 0;
1617 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1618 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1619 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1620 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1622 /* TODO: Add stack limit check */
1624 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1626 if (rc != X86EMUL_CONTINUE)
1629 if (temp_eip & ~0xffff)
1630 return emulate_gp(ctxt, 0);
1632 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1634 if (rc != X86EMUL_CONTINUE)
1637 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1639 if (rc != X86EMUL_CONTINUE)
1642 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1644 if (rc != X86EMUL_CONTINUE)
1647 ctxt->_eip = temp_eip;
1650 if (ctxt->op_bytes == 4)
1651 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1652 else if (ctxt->op_bytes == 2) {
1653 ctxt->eflags &= ~0xffff;
1654 ctxt->eflags |= temp_eflags;
1657 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1658 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1663 static int em_iret(struct x86_emulate_ctxt *ctxt)
1665 switch(ctxt->mode) {
1666 case X86EMUL_MODE_REAL:
1667 return emulate_iret_real(ctxt);
1668 case X86EMUL_MODE_VM86:
1669 case X86EMUL_MODE_PROT16:
1670 case X86EMUL_MODE_PROT32:
1671 case X86EMUL_MODE_PROT64:
1673 /* iret from protected mode unimplemented yet */
1674 return X86EMUL_UNHANDLEABLE;
1678 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1683 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1685 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1686 if (rc != X86EMUL_CONTINUE)
1690 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1691 return X86EMUL_CONTINUE;
1694 static int em_grp1a(struct x86_emulate_ctxt *ctxt)
1696 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->dst.bytes);
1699 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1701 switch (ctxt->modrm_reg) {
1703 emulate_2op_SrcB(ctxt, "rol");
1706 emulate_2op_SrcB(ctxt, "ror");
1709 emulate_2op_SrcB(ctxt, "rcl");
1712 emulate_2op_SrcB(ctxt, "rcr");
1714 case 4: /* sal/shl */
1715 case 6: /* sal/shl */
1716 emulate_2op_SrcB(ctxt, "sal");
1719 emulate_2op_SrcB(ctxt, "shr");
1722 emulate_2op_SrcB(ctxt, "sar");
1725 return X86EMUL_CONTINUE;
1728 static int em_not(struct x86_emulate_ctxt *ctxt)
1730 ctxt->dst.val = ~ctxt->dst.val;
1731 return X86EMUL_CONTINUE;
1734 static int em_neg(struct x86_emulate_ctxt *ctxt)
1736 emulate_1op(ctxt, "neg");
1737 return X86EMUL_CONTINUE;
1740 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
1744 emulate_1op_rax_rdx(ctxt, "mul", ex);
1745 return X86EMUL_CONTINUE;
1748 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
1752 emulate_1op_rax_rdx(ctxt, "imul", ex);
1753 return X86EMUL_CONTINUE;
1756 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1760 emulate_1op_rax_rdx(ctxt, "div", de);
1762 return emulate_de(ctxt);
1763 return X86EMUL_CONTINUE;
1766 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
1770 emulate_1op_rax_rdx(ctxt, "idiv", de);
1772 return emulate_de(ctxt);
1773 return X86EMUL_CONTINUE;
1776 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1778 int rc = X86EMUL_CONTINUE;
1780 switch (ctxt->modrm_reg) {
1782 emulate_1op(ctxt, "inc");
1785 emulate_1op(ctxt, "dec");
1787 case 2: /* call near abs */ {
1789 old_eip = ctxt->_eip;
1790 ctxt->_eip = ctxt->src.val;
1791 ctxt->src.val = old_eip;
1795 case 4: /* jmp abs */
1796 ctxt->_eip = ctxt->src.val;
1798 case 5: /* jmp far */
1799 rc = em_jmp_far(ctxt);
1808 static int em_grp9(struct x86_emulate_ctxt *ctxt)
1810 u64 old = ctxt->dst.orig_val64;
1812 if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
1813 ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
1814 ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1815 ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1816 ctxt->eflags &= ~EFLG_ZF;
1818 ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
1819 (u32) ctxt->regs[VCPU_REGS_RBX];
1821 ctxt->eflags |= EFLG_ZF;
1823 return X86EMUL_CONTINUE;
1826 static int em_ret(struct x86_emulate_ctxt *ctxt)
1828 ctxt->dst.type = OP_REG;
1829 ctxt->dst.addr.reg = &ctxt->_eip;
1830 ctxt->dst.bytes = ctxt->op_bytes;
1831 return em_pop(ctxt);
1834 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
1838 int cpl = ctxt->ops->cpl(ctxt);
1840 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
1841 if (rc != X86EMUL_CONTINUE)
1843 if (ctxt->op_bytes == 4)
1844 ctxt->_eip = (u32)ctxt->_eip;
1845 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1846 if (rc != X86EMUL_CONTINUE)
1848 /* Outer-privilege level return is not implemented */
1849 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
1850 return X86EMUL_UNHANDLEABLE;
1851 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1855 static int em_lseg(struct x86_emulate_ctxt *ctxt)
1857 int seg = ctxt->src2.val;
1861 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1863 rc = load_segment_descriptor(ctxt, sel, seg);
1864 if (rc != X86EMUL_CONTINUE)
1867 ctxt->dst.val = ctxt->src.val;
1872 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1873 struct desc_struct *cs, struct desc_struct *ss)
1877 memset(cs, 0, sizeof(struct desc_struct));
1878 ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1879 memset(ss, 0, sizeof(struct desc_struct));
1881 cs->l = 0; /* will be adjusted later */
1882 set_desc_base(cs, 0); /* flat segment */
1883 cs->g = 1; /* 4kb granularity */
1884 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1885 cs->type = 0x0b; /* Read, Execute, Accessed */
1887 cs->dpl = 0; /* will be adjusted later */
1891 set_desc_base(ss, 0); /* flat segment */
1892 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1893 ss->g = 1; /* 4kb granularity */
1895 ss->type = 0x03; /* Read/Write, Accessed */
1896 ss->d = 1; /* 32bit stack segment */
1901 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
1903 struct x86_emulate_ops *ops = ctxt->ops;
1904 u32 eax, ebx, ecx, edx;
1907 * syscall should always be enabled in longmode - so only become
1908 * vendor specific (cpuid) if other modes are active...
1910 if (ctxt->mode == X86EMUL_MODE_PROT64)
1915 if (ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx)) {
1917 * Intel ("GenuineIntel")
1918 * remark: Intel CPUs only support "syscall" in 64bit
1919 * longmode. Also an 64bit guest with a
1920 * 32bit compat-app running will #UD !! While this
1921 * behaviour can be fixed (by emulating) into AMD
1922 * response - CPUs of AMD can't behave like Intel.
1924 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
1925 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
1926 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
1929 /* AMD ("AuthenticAMD") */
1930 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
1931 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
1932 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
1935 /* AMD ("AMDisbetter!") */
1936 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
1937 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
1938 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
1942 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
1946 static int em_syscall(struct x86_emulate_ctxt *ctxt)
1948 struct x86_emulate_ops *ops = ctxt->ops;
1949 struct desc_struct cs, ss;
1954 /* syscall is not available in real mode */
1955 if (ctxt->mode == X86EMUL_MODE_REAL ||
1956 ctxt->mode == X86EMUL_MODE_VM86)
1957 return emulate_ud(ctxt);
1959 if (!(em_syscall_is_enabled(ctxt)))
1960 return emulate_ud(ctxt);
1962 ops->get_msr(ctxt, MSR_EFER, &efer);
1963 setup_syscalls_segments(ctxt, &cs, &ss);
1965 if (!(efer & EFER_SCE))
1966 return emulate_ud(ctxt);
1968 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1970 cs_sel = (u16)(msr_data & 0xfffc);
1971 ss_sel = (u16)(msr_data + 8);
1973 if (efer & EFER_LMA) {
1977 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1978 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1980 ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
1981 if (efer & EFER_LMA) {
1982 #ifdef CONFIG_X86_64
1983 ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1986 ctxt->mode == X86EMUL_MODE_PROT64 ?
1987 MSR_LSTAR : MSR_CSTAR, &msr_data);
1988 ctxt->_eip = msr_data;
1990 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1991 ctxt->eflags &= ~(msr_data | EFLG_RF);
1995 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1996 ctxt->_eip = (u32)msr_data;
1998 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2001 return X86EMUL_CONTINUE;
2004 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2006 struct x86_emulate_ops *ops = ctxt->ops;
2007 struct desc_struct cs, ss;
2012 ops->get_msr(ctxt, MSR_EFER, &efer);
2013 /* inject #GP if in real mode */
2014 if (ctxt->mode == X86EMUL_MODE_REAL)
2015 return emulate_gp(ctxt, 0);
2017 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2018 * Therefore, we inject an #UD.
2020 if (ctxt->mode == X86EMUL_MODE_PROT64)
2021 return emulate_ud(ctxt);
2023 setup_syscalls_segments(ctxt, &cs, &ss);
2025 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2026 switch (ctxt->mode) {
2027 case X86EMUL_MODE_PROT32:
2028 if ((msr_data & 0xfffc) == 0x0)
2029 return emulate_gp(ctxt, 0);
2031 case X86EMUL_MODE_PROT64:
2032 if (msr_data == 0x0)
2033 return emulate_gp(ctxt, 0);
2037 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2038 cs_sel = (u16)msr_data;
2039 cs_sel &= ~SELECTOR_RPL_MASK;
2040 ss_sel = cs_sel + 8;
2041 ss_sel &= ~SELECTOR_RPL_MASK;
2042 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2047 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2048 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2050 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2051 ctxt->_eip = msr_data;
2053 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2054 ctxt->regs[VCPU_REGS_RSP] = msr_data;
2056 return X86EMUL_CONTINUE;
2059 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2061 struct x86_emulate_ops *ops = ctxt->ops;
2062 struct desc_struct cs, ss;
2065 u16 cs_sel = 0, ss_sel = 0;
2067 /* inject #GP if in real mode or Virtual 8086 mode */
2068 if (ctxt->mode == X86EMUL_MODE_REAL ||
2069 ctxt->mode == X86EMUL_MODE_VM86)
2070 return emulate_gp(ctxt, 0);
2072 setup_syscalls_segments(ctxt, &cs, &ss);
2074 if ((ctxt->rex_prefix & 0x8) != 0x0)
2075 usermode = X86EMUL_MODE_PROT64;
2077 usermode = X86EMUL_MODE_PROT32;
2081 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2083 case X86EMUL_MODE_PROT32:
2084 cs_sel = (u16)(msr_data + 16);
2085 if ((msr_data & 0xfffc) == 0x0)
2086 return emulate_gp(ctxt, 0);
2087 ss_sel = (u16)(msr_data + 24);
2089 case X86EMUL_MODE_PROT64:
2090 cs_sel = (u16)(msr_data + 32);
2091 if (msr_data == 0x0)
2092 return emulate_gp(ctxt, 0);
2093 ss_sel = cs_sel + 8;
2098 cs_sel |= SELECTOR_RPL_MASK;
2099 ss_sel |= SELECTOR_RPL_MASK;
2101 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2102 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2104 ctxt->_eip = ctxt->regs[VCPU_REGS_RDX];
2105 ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX];
2107 return X86EMUL_CONTINUE;
2110 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2113 if (ctxt->mode == X86EMUL_MODE_REAL)
2115 if (ctxt->mode == X86EMUL_MODE_VM86)
2117 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2118 return ctxt->ops->cpl(ctxt) > iopl;
2121 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2124 struct x86_emulate_ops *ops = ctxt->ops;
2125 struct desc_struct tr_seg;
2128 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2129 unsigned mask = (1 << len) - 1;
2132 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2135 if (desc_limit_scaled(&tr_seg) < 103)
2137 base = get_desc_base(&tr_seg);
2138 #ifdef CONFIG_X86_64
2139 base |= ((u64)base3) << 32;
2141 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2142 if (r != X86EMUL_CONTINUE)
2144 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2146 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2147 if (r != X86EMUL_CONTINUE)
2149 if ((perm >> bit_idx) & mask)
2154 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2160 if (emulator_bad_iopl(ctxt))
2161 if (!emulator_io_port_access_allowed(ctxt, port, len))
2164 ctxt->perm_ok = true;
2169 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2170 struct tss_segment_16 *tss)
2172 tss->ip = ctxt->_eip;
2173 tss->flag = ctxt->eflags;
2174 tss->ax = ctxt->regs[VCPU_REGS_RAX];
2175 tss->cx = ctxt->regs[VCPU_REGS_RCX];
2176 tss->dx = ctxt->regs[VCPU_REGS_RDX];
2177 tss->bx = ctxt->regs[VCPU_REGS_RBX];
2178 tss->sp = ctxt->regs[VCPU_REGS_RSP];
2179 tss->bp = ctxt->regs[VCPU_REGS_RBP];
2180 tss->si = ctxt->regs[VCPU_REGS_RSI];
2181 tss->di = ctxt->regs[VCPU_REGS_RDI];
2183 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2184 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2185 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2186 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2187 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2190 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2191 struct tss_segment_16 *tss)
2195 ctxt->_eip = tss->ip;
2196 ctxt->eflags = tss->flag | 2;
2197 ctxt->regs[VCPU_REGS_RAX] = tss->ax;
2198 ctxt->regs[VCPU_REGS_RCX] = tss->cx;
2199 ctxt->regs[VCPU_REGS_RDX] = tss->dx;
2200 ctxt->regs[VCPU_REGS_RBX] = tss->bx;
2201 ctxt->regs[VCPU_REGS_RSP] = tss->sp;
2202 ctxt->regs[VCPU_REGS_RBP] = tss->bp;
2203 ctxt->regs[VCPU_REGS_RSI] = tss->si;
2204 ctxt->regs[VCPU_REGS_RDI] = tss->di;
2207 * SDM says that segment selectors are loaded before segment
2210 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2211 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2212 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2213 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2214 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2217 * Now load segment descriptors. If fault happenes at this stage
2218 * it is handled in a context of new task
2220 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2221 if (ret != X86EMUL_CONTINUE)
2223 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2224 if (ret != X86EMUL_CONTINUE)
2226 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2227 if (ret != X86EMUL_CONTINUE)
2229 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2230 if (ret != X86EMUL_CONTINUE)
2232 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2233 if (ret != X86EMUL_CONTINUE)
2236 return X86EMUL_CONTINUE;
2239 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2240 u16 tss_selector, u16 old_tss_sel,
2241 ulong old_tss_base, struct desc_struct *new_desc)
2243 struct x86_emulate_ops *ops = ctxt->ops;
2244 struct tss_segment_16 tss_seg;
2246 u32 new_tss_base = get_desc_base(new_desc);
2248 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2250 if (ret != X86EMUL_CONTINUE)
2251 /* FIXME: need to provide precise fault address */
2254 save_state_to_tss16(ctxt, &tss_seg);
2256 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2258 if (ret != X86EMUL_CONTINUE)
2259 /* FIXME: need to provide precise fault address */
2262 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2264 if (ret != X86EMUL_CONTINUE)
2265 /* FIXME: need to provide precise fault address */
2268 if (old_tss_sel != 0xffff) {
2269 tss_seg.prev_task_link = old_tss_sel;
2271 ret = ops->write_std(ctxt, new_tss_base,
2272 &tss_seg.prev_task_link,
2273 sizeof tss_seg.prev_task_link,
2275 if (ret != X86EMUL_CONTINUE)
2276 /* FIXME: need to provide precise fault address */
2280 return load_state_from_tss16(ctxt, &tss_seg);
2283 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2284 struct tss_segment_32 *tss)
2286 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2287 tss->eip = ctxt->_eip;
2288 tss->eflags = ctxt->eflags;
2289 tss->eax = ctxt->regs[VCPU_REGS_RAX];
2290 tss->ecx = ctxt->regs[VCPU_REGS_RCX];
2291 tss->edx = ctxt->regs[VCPU_REGS_RDX];
2292 tss->ebx = ctxt->regs[VCPU_REGS_RBX];
2293 tss->esp = ctxt->regs[VCPU_REGS_RSP];
2294 tss->ebp = ctxt->regs[VCPU_REGS_RBP];
2295 tss->esi = ctxt->regs[VCPU_REGS_RSI];
2296 tss->edi = ctxt->regs[VCPU_REGS_RDI];
2298 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2299 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2300 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2301 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2302 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2303 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2304 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2307 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2308 struct tss_segment_32 *tss)
2312 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2313 return emulate_gp(ctxt, 0);
2314 ctxt->_eip = tss->eip;
2315 ctxt->eflags = tss->eflags | 2;
2316 ctxt->regs[VCPU_REGS_RAX] = tss->eax;
2317 ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
2318 ctxt->regs[VCPU_REGS_RDX] = tss->edx;
2319 ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
2320 ctxt->regs[VCPU_REGS_RSP] = tss->esp;
2321 ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
2322 ctxt->regs[VCPU_REGS_RSI] = tss->esi;
2323 ctxt->regs[VCPU_REGS_RDI] = tss->edi;
2326 * SDM says that segment selectors are loaded before segment
2329 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2330 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2331 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2332 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2333 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2334 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2335 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2338 * Now load segment descriptors. If fault happenes at this stage
2339 * it is handled in a context of new task
2341 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2342 if (ret != X86EMUL_CONTINUE)
2344 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2345 if (ret != X86EMUL_CONTINUE)
2347 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2348 if (ret != X86EMUL_CONTINUE)
2350 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2351 if (ret != X86EMUL_CONTINUE)
2353 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2354 if (ret != X86EMUL_CONTINUE)
2356 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2357 if (ret != X86EMUL_CONTINUE)
2359 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2360 if (ret != X86EMUL_CONTINUE)
2363 return X86EMUL_CONTINUE;
2366 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2367 u16 tss_selector, u16 old_tss_sel,
2368 ulong old_tss_base, struct desc_struct *new_desc)
2370 struct x86_emulate_ops *ops = ctxt->ops;
2371 struct tss_segment_32 tss_seg;
2373 u32 new_tss_base = get_desc_base(new_desc);
2375 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2377 if (ret != X86EMUL_CONTINUE)
2378 /* FIXME: need to provide precise fault address */
2381 save_state_to_tss32(ctxt, &tss_seg);
2383 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2385 if (ret != X86EMUL_CONTINUE)
2386 /* FIXME: need to provide precise fault address */
2389 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2391 if (ret != X86EMUL_CONTINUE)
2392 /* FIXME: need to provide precise fault address */
2395 if (old_tss_sel != 0xffff) {
2396 tss_seg.prev_task_link = old_tss_sel;
2398 ret = ops->write_std(ctxt, new_tss_base,
2399 &tss_seg.prev_task_link,
2400 sizeof tss_seg.prev_task_link,
2402 if (ret != X86EMUL_CONTINUE)
2403 /* FIXME: need to provide precise fault address */
2407 return load_state_from_tss32(ctxt, &tss_seg);
2410 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2411 u16 tss_selector, int reason,
2412 bool has_error_code, u32 error_code)
2414 struct x86_emulate_ops *ops = ctxt->ops;
2415 struct desc_struct curr_tss_desc, next_tss_desc;
2417 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2418 ulong old_tss_base =
2419 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2422 /* FIXME: old_tss_base == ~0 ? */
2424 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2425 if (ret != X86EMUL_CONTINUE)
2427 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2428 if (ret != X86EMUL_CONTINUE)
2431 /* FIXME: check that next_tss_desc is tss */
2433 if (reason != TASK_SWITCH_IRET) {
2434 if ((tss_selector & 3) > next_tss_desc.dpl ||
2435 ops->cpl(ctxt) > next_tss_desc.dpl)
2436 return emulate_gp(ctxt, 0);
2439 desc_limit = desc_limit_scaled(&next_tss_desc);
2440 if (!next_tss_desc.p ||
2441 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2442 desc_limit < 0x2b)) {
2443 emulate_ts(ctxt, tss_selector & 0xfffc);
2444 return X86EMUL_PROPAGATE_FAULT;
2447 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2448 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2449 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2452 if (reason == TASK_SWITCH_IRET)
2453 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2455 /* set back link to prev task only if NT bit is set in eflags
2456 note that old_tss_sel is not used afetr this point */
2457 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2458 old_tss_sel = 0xffff;
2460 if (next_tss_desc.type & 8)
2461 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2462 old_tss_base, &next_tss_desc);
2464 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2465 old_tss_base, &next_tss_desc);
2466 if (ret != X86EMUL_CONTINUE)
2469 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2470 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2472 if (reason != TASK_SWITCH_IRET) {
2473 next_tss_desc.type |= (1 << 1); /* set busy flag */
2474 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2477 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2478 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2480 if (has_error_code) {
2481 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2482 ctxt->lock_prefix = 0;
2483 ctxt->src.val = (unsigned long) error_code;
2484 ret = em_push(ctxt);
2490 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2491 u16 tss_selector, int reason,
2492 bool has_error_code, u32 error_code)
2496 ctxt->_eip = ctxt->eip;
2497 ctxt->dst.type = OP_NONE;
2499 rc = emulator_do_task_switch(ctxt, tss_selector, reason,
2500 has_error_code, error_code);
2502 if (rc == X86EMUL_CONTINUE)
2503 ctxt->eip = ctxt->_eip;
2505 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2508 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2509 int reg, struct operand *op)
2511 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2513 register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
2514 op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
2515 op->addr.mem.seg = seg;
2518 static int em_das(struct x86_emulate_ctxt *ctxt)
2521 bool af, cf, old_cf;
2523 cf = ctxt->eflags & X86_EFLAGS_CF;
2529 af = ctxt->eflags & X86_EFLAGS_AF;
2530 if ((al & 0x0f) > 9 || af) {
2532 cf = old_cf | (al >= 250);
2537 if (old_al > 0x99 || old_cf) {
2543 /* Set PF, ZF, SF */
2544 ctxt->src.type = OP_IMM;
2546 ctxt->src.bytes = 1;
2547 emulate_2op_SrcV(ctxt, "or");
2548 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2550 ctxt->eflags |= X86_EFLAGS_CF;
2552 ctxt->eflags |= X86_EFLAGS_AF;
2553 return X86EMUL_CONTINUE;
2556 static int em_call(struct x86_emulate_ctxt *ctxt)
2558 long rel = ctxt->src.val;
2560 ctxt->src.val = (unsigned long)ctxt->_eip;
2562 return em_push(ctxt);
2565 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2571 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2572 old_eip = ctxt->_eip;
2574 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2575 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2576 return X86EMUL_CONTINUE;
2579 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2581 ctxt->src.val = old_cs;
2583 if (rc != X86EMUL_CONTINUE)
2586 ctxt->src.val = old_eip;
2587 return em_push(ctxt);
2590 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2594 ctxt->dst.type = OP_REG;
2595 ctxt->dst.addr.reg = &ctxt->_eip;
2596 ctxt->dst.bytes = ctxt->op_bytes;
2597 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2598 if (rc != X86EMUL_CONTINUE)
2600 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], ctxt->src.val);
2601 return X86EMUL_CONTINUE;
2604 static int em_add(struct x86_emulate_ctxt *ctxt)
2606 emulate_2op_SrcV(ctxt, "add");
2607 return X86EMUL_CONTINUE;
2610 static int em_or(struct x86_emulate_ctxt *ctxt)
2612 emulate_2op_SrcV(ctxt, "or");
2613 return X86EMUL_CONTINUE;
2616 static int em_adc(struct x86_emulate_ctxt *ctxt)
2618 emulate_2op_SrcV(ctxt, "adc");
2619 return X86EMUL_CONTINUE;
2622 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2624 emulate_2op_SrcV(ctxt, "sbb");
2625 return X86EMUL_CONTINUE;
2628 static int em_and(struct x86_emulate_ctxt *ctxt)
2630 emulate_2op_SrcV(ctxt, "and");
2631 return X86EMUL_CONTINUE;
2634 static int em_sub(struct x86_emulate_ctxt *ctxt)
2636 emulate_2op_SrcV(ctxt, "sub");
2637 return X86EMUL_CONTINUE;
2640 static int em_xor(struct x86_emulate_ctxt *ctxt)
2642 emulate_2op_SrcV(ctxt, "xor");
2643 return X86EMUL_CONTINUE;
2646 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2648 emulate_2op_SrcV(ctxt, "cmp");
2649 /* Disable writeback. */
2650 ctxt->dst.type = OP_NONE;
2651 return X86EMUL_CONTINUE;
2654 static int em_test(struct x86_emulate_ctxt *ctxt)
2656 emulate_2op_SrcV(ctxt, "test");
2657 /* Disable writeback. */
2658 ctxt->dst.type = OP_NONE;
2659 return X86EMUL_CONTINUE;
2662 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2664 /* Write back the register source. */
2665 ctxt->src.val = ctxt->dst.val;
2666 write_register_operand(&ctxt->src);
2668 /* Write back the memory destination with implicit LOCK prefix. */
2669 ctxt->dst.val = ctxt->src.orig_val;
2670 ctxt->lock_prefix = 1;
2671 return X86EMUL_CONTINUE;
2674 static int em_imul(struct x86_emulate_ctxt *ctxt)
2676 emulate_2op_SrcV_nobyte(ctxt, "imul");
2677 return X86EMUL_CONTINUE;
2680 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2682 ctxt->dst.val = ctxt->src2.val;
2683 return em_imul(ctxt);
2686 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2688 ctxt->dst.type = OP_REG;
2689 ctxt->dst.bytes = ctxt->src.bytes;
2690 ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
2691 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2693 return X86EMUL_CONTINUE;
2696 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2700 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2701 ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
2702 ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
2703 return X86EMUL_CONTINUE;
2706 static int em_mov(struct x86_emulate_ctxt *ctxt)
2708 ctxt->dst.val = ctxt->src.val;
2709 return X86EMUL_CONTINUE;
2712 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
2714 if (ctxt->modrm_reg > VCPU_SREG_GS)
2715 return emulate_ud(ctxt);
2717 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
2718 return X86EMUL_CONTINUE;
2721 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
2723 u16 sel = ctxt->src.val;
2725 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
2726 return emulate_ud(ctxt);
2728 if (ctxt->modrm_reg == VCPU_SREG_SS)
2729 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2731 /* Disable writeback. */
2732 ctxt->dst.type = OP_NONE;
2733 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
2736 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2738 memcpy(&ctxt->dst.vec_val, &ctxt->src.vec_val, ctxt->op_bytes);
2739 return X86EMUL_CONTINUE;
2742 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2747 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
2748 if (rc == X86EMUL_CONTINUE)
2749 ctxt->ops->invlpg(ctxt, linear);
2750 /* Disable writeback. */
2751 ctxt->dst.type = OP_NONE;
2752 return X86EMUL_CONTINUE;
2755 static int em_clts(struct x86_emulate_ctxt *ctxt)
2759 cr0 = ctxt->ops->get_cr(ctxt, 0);
2761 ctxt->ops->set_cr(ctxt, 0, cr0);
2762 return X86EMUL_CONTINUE;
2765 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
2769 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
2770 return X86EMUL_UNHANDLEABLE;
2772 rc = ctxt->ops->fix_hypercall(ctxt);
2773 if (rc != X86EMUL_CONTINUE)
2776 /* Let the processor re-execute the fixed hypercall */
2777 ctxt->_eip = ctxt->eip;
2778 /* Disable writeback. */
2779 ctxt->dst.type = OP_NONE;
2780 return X86EMUL_CONTINUE;
2783 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
2785 struct desc_ptr desc_ptr;
2788 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2789 &desc_ptr.size, &desc_ptr.address,
2791 if (rc != X86EMUL_CONTINUE)
2793 ctxt->ops->set_gdt(ctxt, &desc_ptr);
2794 /* Disable writeback. */
2795 ctxt->dst.type = OP_NONE;
2796 return X86EMUL_CONTINUE;
2799 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2803 rc = ctxt->ops->fix_hypercall(ctxt);
2805 /* Disable writeback. */
2806 ctxt->dst.type = OP_NONE;
2810 static int em_lidt(struct x86_emulate_ctxt *ctxt)
2812 struct desc_ptr desc_ptr;
2815 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2816 &desc_ptr.size, &desc_ptr.address,
2818 if (rc != X86EMUL_CONTINUE)
2820 ctxt->ops->set_idt(ctxt, &desc_ptr);
2821 /* Disable writeback. */
2822 ctxt->dst.type = OP_NONE;
2823 return X86EMUL_CONTINUE;
2826 static int em_smsw(struct x86_emulate_ctxt *ctxt)
2828 ctxt->dst.bytes = 2;
2829 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
2830 return X86EMUL_CONTINUE;
2833 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
2835 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2836 | (ctxt->src.val & 0x0f));
2837 ctxt->dst.type = OP_NONE;
2838 return X86EMUL_CONTINUE;
2841 static int em_loop(struct x86_emulate_ctxt *ctxt)
2843 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
2844 if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
2845 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
2846 jmp_rel(ctxt, ctxt->src.val);
2848 return X86EMUL_CONTINUE;
2851 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
2853 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
2854 jmp_rel(ctxt, ctxt->src.val);
2856 return X86EMUL_CONTINUE;
2859 static int em_cli(struct x86_emulate_ctxt *ctxt)
2861 if (emulator_bad_iopl(ctxt))
2862 return emulate_gp(ctxt, 0);
2864 ctxt->eflags &= ~X86_EFLAGS_IF;
2865 return X86EMUL_CONTINUE;
2868 static int em_sti(struct x86_emulate_ctxt *ctxt)
2870 if (emulator_bad_iopl(ctxt))
2871 return emulate_gp(ctxt, 0);
2873 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
2874 ctxt->eflags |= X86_EFLAGS_IF;
2875 return X86EMUL_CONTINUE;
2878 static bool valid_cr(int nr)
2890 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2892 if (!valid_cr(ctxt->modrm_reg))
2893 return emulate_ud(ctxt);
2895 return X86EMUL_CONTINUE;
2898 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2900 u64 new_val = ctxt->src.val64;
2901 int cr = ctxt->modrm_reg;
2904 static u64 cr_reserved_bits[] = {
2905 0xffffffff00000000ULL,
2906 0, 0, 0, /* CR3 checked later */
2913 return emulate_ud(ctxt);
2915 if (new_val & cr_reserved_bits[cr])
2916 return emulate_gp(ctxt, 0);
2921 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2922 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2923 return emulate_gp(ctxt, 0);
2925 cr4 = ctxt->ops->get_cr(ctxt, 4);
2926 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2928 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2929 !(cr4 & X86_CR4_PAE))
2930 return emulate_gp(ctxt, 0);
2937 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2938 if (efer & EFER_LMA)
2939 rsvd = CR3_L_MODE_RESERVED_BITS;
2940 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
2941 rsvd = CR3_PAE_RESERVED_BITS;
2942 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
2943 rsvd = CR3_NONPAE_RESERVED_BITS;
2946 return emulate_gp(ctxt, 0);
2953 cr4 = ctxt->ops->get_cr(ctxt, 4);
2954 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2956 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2957 return emulate_gp(ctxt, 0);
2963 return X86EMUL_CONTINUE;
2966 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2970 ctxt->ops->get_dr(ctxt, 7, &dr7);
2972 /* Check if DR7.Global_Enable is set */
2973 return dr7 & (1 << 13);
2976 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2978 int dr = ctxt->modrm_reg;
2982 return emulate_ud(ctxt);
2984 cr4 = ctxt->ops->get_cr(ctxt, 4);
2985 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2986 return emulate_ud(ctxt);
2988 if (check_dr7_gd(ctxt))
2989 return emulate_db(ctxt);
2991 return X86EMUL_CONTINUE;
2994 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2996 u64 new_val = ctxt->src.val64;
2997 int dr = ctxt->modrm_reg;
2999 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3000 return emulate_gp(ctxt, 0);
3002 return check_dr_read(ctxt);
3005 static int check_svme(struct x86_emulate_ctxt *ctxt)
3009 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3011 if (!(efer & EFER_SVME))
3012 return emulate_ud(ctxt);
3014 return X86EMUL_CONTINUE;
3017 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3019 u64 rax = ctxt->regs[VCPU_REGS_RAX];
3021 /* Valid physical address? */
3022 if (rax & 0xffff000000000000ULL)
3023 return emulate_gp(ctxt, 0);
3025 return check_svme(ctxt);
3028 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3030 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3032 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3033 return emulate_ud(ctxt);
3035 return X86EMUL_CONTINUE;
3038 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3040 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3041 u64 rcx = ctxt->regs[VCPU_REGS_RCX];
3043 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3045 return emulate_gp(ctxt, 0);
3047 return X86EMUL_CONTINUE;
3050 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3052 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3053 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3054 return emulate_gp(ctxt, 0);
3056 return X86EMUL_CONTINUE;
3059 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3061 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3062 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3063 return emulate_gp(ctxt, 0);
3065 return X86EMUL_CONTINUE;
3068 #define D(_y) { .flags = (_y) }
3069 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3070 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3071 .check_perm = (_p) }
3073 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3074 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
3075 #define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
3076 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3077 #define II(_f, _e, _i) \
3078 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3079 #define IIP(_f, _e, _i, _p) \
3080 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3081 .check_perm = (_p) }
3082 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3084 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3085 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3086 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3088 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3089 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3090 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3092 static struct opcode group7_rm1[] = {
3093 DI(SrcNone | ModRM | Priv, monitor),
3094 DI(SrcNone | ModRM | Priv, mwait),
3098 static struct opcode group7_rm3[] = {
3099 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
3100 II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
3101 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
3102 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
3103 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
3104 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
3105 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
3106 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
3109 static struct opcode group7_rm7[] = {
3111 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
3115 static struct opcode group1[] = {
3126 static struct opcode group1A[] = {
3127 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
3130 static struct opcode group3[] = {
3131 I(DstMem | SrcImm | ModRM, em_test),
3132 I(DstMem | SrcImm | ModRM, em_test),
3133 I(DstMem | SrcNone | ModRM | Lock, em_not),
3134 I(DstMem | SrcNone | ModRM | Lock, em_neg),
3135 I(SrcMem | ModRM, em_mul_ex),
3136 I(SrcMem | ModRM, em_imul_ex),
3137 I(SrcMem | ModRM, em_div_ex),
3138 I(SrcMem | ModRM, em_idiv_ex),
3141 static struct opcode group4[] = {
3142 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),