1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affilates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
26 #include <public/xen.h>
27 #define DPRINTF(_f, _a ...) printf(_f , ## _a)
29 #include <linux/kvm_host.h>
30 #include "kvm_cache_regs.h"
31 #define DPRINTF(x...) do {} while (0)
33 #include <linux/module.h>
34 #include <asm/kvm_emulate.h>
40 * Opcode effective-address decode tables.
41 * Note that we only emulate instructions that have at least one memory
42 * operand (excluding implicit stack references). We assume that stack
43 * references and instruction fetches will never occur in special memory
44 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
48 /* Operand sizes: 8-bit operands or specified/overridden size. */
49 #define ByteOp (1<<16) /* 8-bit operands. */
50 /* Destination operand type. */
51 #define ImplicitOps (1<<17) /* Implicit in opcode. No generic decode. */
52 #define DstReg (2<<17) /* Register operand. */
53 #define DstMem (3<<17) /* Memory operand. */
54 #define DstAcc (4<<17) /* Destination Accumulator */
55 #define DstDI (5<<17) /* Destination is in ES:(E)DI */
56 #define DstMem64 (6<<17) /* 64bit memory operand */
57 #define DstMask (7<<17)
58 /* Source operand type. */
59 #define SrcNone (0<<4) /* No source operand. */
60 #define SrcImplicit (0<<4) /* Source operand is implicit in the opcode. */
61 #define SrcReg (1<<4) /* Register operand. */
62 #define SrcMem (2<<4) /* Memory operand. */
63 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
64 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
65 #define SrcImm (5<<4) /* Immediate operand. */
66 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
67 #define SrcOne (7<<4) /* Implied '1' */
68 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
69 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
70 #define SrcSI (0xa<<4) /* Source is in the DS:RSI */
71 #define SrcImmFAddr (0xb<<4) /* Source is immediate far address */
72 #define SrcMemFAddr (0xc<<4) /* Source is far address in memory */
73 #define SrcAcc (0xd<<4) /* Source Accumulator */
74 #define SrcMask (0xf<<4)
75 /* Generic ModRM decode. */
77 /* Destination is only written; never read. */
80 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
81 #define String (1<<12) /* String instruction (rep capable) */
82 #define Stack (1<<13) /* Stack instruction (push/pop) */
83 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
84 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
85 #define GroupMask 0x0f /* Group number stored in bits 0:3 */
87 #define Undefined (1<<25) /* No Such Instruction */
88 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
89 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
91 /* Source 2 operand type */
92 #define Src2None (0<<29)
93 #define Src2CL (1<<29)
94 #define Src2ImmByte (2<<29)
95 #define Src2One (3<<29)
96 #define Src2Mask (7<<29)
99 #define X3(x) X2(x), x
100 #define X4(x) X2(x), X2(x)
101 #define X5(x) X4(x), x
102 #define X6(x) X4(x), X2(x)
103 #define X7(x) X4(x), X3(x)
104 #define X8(x) X4(x), X4(x)
105 #define X16(x) X8(x), X8(x)
108 Group1, Group1A, Group3, Group4, Group5, Group7, Group8, Group9,
115 static struct opcode opcode_table[256] = {
117 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
118 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
119 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
120 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
122 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
123 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
124 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
125 ImplicitOps | Stack | No64, 0,
127 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
128 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
129 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
130 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
132 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
133 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
134 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
135 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
137 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
138 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
139 ByteOp | DstAcc | SrcImmByte, DstAcc | SrcImm, 0, 0,
141 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
142 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
143 ByteOp | DstAcc | SrcImmByte, DstAcc | SrcImm, 0, 0,
145 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
146 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
147 ByteOp | DstAcc | SrcImmByte, DstAcc | SrcImm, 0, 0,
149 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
150 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
151 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
160 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
161 0, DstReg | SrcMem32 | ModRM | Mov /* movsxd (x86/64) */ ,
164 SrcImm | Mov | Stack, 0, SrcImmByte | Mov | Stack, 0,
165 DstDI | ByteOp | Mov | String, DstDI | Mov | String, /* insb, insw/insd */
166 SrcSI | ByteOp | ImplicitOps | String, SrcSI | ImplicitOps | String, /* outsb, outsw/outsd */
170 ByteOp | DstMem | SrcImm | ModRM | Group | Group1,
171 DstMem | SrcImm | ModRM | Group | Group1,
172 ByteOp | DstMem | SrcImm | ModRM | No64 | Group | Group1,
173 DstMem | SrcImmByte | ModRM | Group | Group1,
174 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
175 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
177 ByteOp | DstMem | SrcReg | ModRM | Mov, DstMem | SrcReg | ModRM | Mov,
178 ByteOp | DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
179 DstMem | SrcNone | ModRM | Mov, ModRM | DstReg,
180 ImplicitOps | SrcMem16 | ModRM, Group | Group1A,
182 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
184 0, 0, SrcImmFAddr | No64, 0,
185 ImplicitOps | Stack, ImplicitOps | Stack, 0, 0,
187 ByteOp | DstAcc | SrcMem | Mov | MemAbs, DstAcc | SrcMem | Mov | MemAbs,
188 ByteOp | DstMem | SrcAcc | Mov | MemAbs, DstMem | SrcAcc | Mov | MemAbs,
189 ByteOp | SrcSI | DstDI | Mov | String, SrcSI | DstDI | Mov | String,
190 ByteOp | SrcSI | DstDI | String, SrcSI | DstDI | String,
192 DstAcc | SrcImmByte | ByteOp, DstAcc | SrcImm, ByteOp | DstDI | Mov | String, DstDI | Mov | String,
193 ByteOp | SrcSI | DstAcc | Mov | String, SrcSI | DstAcc | Mov | String,
194 ByteOp | DstDI | String, DstDI | String,
196 X8(ByteOp | DstReg | SrcImm | Mov),
198 X8(DstReg | SrcImm | Mov),
200 ByteOp | DstMem | SrcImm | ModRM, DstMem | SrcImmByte | ModRM,
201 0, ImplicitOps | Stack, 0, 0,
202 ByteOp | DstMem | SrcImm | ModRM | Mov, DstMem | SrcImm | ModRM | Mov,
204 0, 0, 0, ImplicitOps | Stack,
205 ImplicitOps, SrcImmByte, ImplicitOps | No64, ImplicitOps,
207 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
208 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
211 0, 0, 0, 0, 0, 0, 0, 0,
214 ByteOp | SrcImmUByte | DstAcc, SrcImmUByte | DstAcc,
215 ByteOp | SrcImmUByte | DstAcc, SrcImmUByte | DstAcc,
217 SrcImm | Stack, SrcImm | ImplicitOps,
218 SrcImmFAddr | No64, SrcImmByte | ImplicitOps,
219 SrcNone | ByteOp | DstAcc, SrcNone | DstAcc,
220 SrcNone | ByteOp | DstAcc, SrcNone | DstAcc,
223 ImplicitOps | Priv, ImplicitOps, ByteOp | Group | Group3, Group | Group3,
225 ImplicitOps, 0, ImplicitOps, ImplicitOps,
226 ImplicitOps, ImplicitOps, Group | Group4, Group | Group5,
229 static struct opcode twobyte_table[256] = {
231 0, Group | GroupDual | Group7, 0, 0,
232 0, ImplicitOps, ImplicitOps | Priv, 0,
233 ImplicitOps | Priv, ImplicitOps | Priv, 0, 0,
234 0, ImplicitOps | ModRM, 0, 0,
236 0, 0, 0, 0, 0, 0, 0, 0, ImplicitOps | ModRM, 0, 0, 0, 0, 0, 0, 0,
238 ModRM | ImplicitOps | Priv, ModRM | Priv,
239 ModRM | ImplicitOps | Priv, ModRM | Priv,
241 0, 0, 0, 0, 0, 0, 0, 0,
243 ImplicitOps | Priv, 0, ImplicitOps | Priv, 0,
244 ImplicitOps, ImplicitOps | Priv, 0, 0,
245 0, 0, 0, 0, 0, 0, 0, 0,
247 X16(DstReg | SrcMem | ModRM | Mov),
249 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
251 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
253 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
257 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
259 ImplicitOps | Stack, ImplicitOps | Stack,
260 0, DstMem | SrcReg | ModRM | BitOp,
261 DstMem | SrcReg | Src2ImmByte | ModRM,
262 DstMem | SrcReg | Src2CL | ModRM, 0, 0,
264 ImplicitOps | Stack, ImplicitOps | Stack,
265 0, DstMem | SrcReg | ModRM | BitOp | Lock,
266 DstMem | SrcReg | Src2ImmByte | ModRM,
267 DstMem | SrcReg | Src2CL | ModRM,
270 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
271 0, DstMem | SrcReg | ModRM | BitOp | Lock,
272 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
273 DstReg | SrcMem16 | ModRM | Mov,
276 Group | Group8, DstMem | SrcReg | ModRM | BitOp | Lock,
277 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
278 DstReg | SrcMem16 | ModRM | Mov,
280 0, 0, 0, DstMem | SrcReg | ModRM | Mov,
281 0, 0, 0, Group | GroupDual | Group9,
282 0, 0, 0, 0, 0, 0, 0, 0,
284 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
286 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
288 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
291 static struct opcode group_table[] = {
295 DstMem | SrcNone | ModRM | Mov | Stack, 0, 0, 0, 0, 0, 0, 0,
297 DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
298 DstMem | SrcNone | ModRM | Lock, DstMem | SrcNone | ModRM | Lock,
301 ByteOp | DstMem | SrcNone | ModRM | Lock, ByteOp | DstMem | SrcNone | ModRM | Lock,
304 DstMem | SrcNone | ModRM | Lock, DstMem | SrcNone | ModRM | Lock,
305 SrcMem | ModRM | Stack, 0,
306 SrcMem | ModRM | Stack, SrcMemFAddr | ModRM | ImplicitOps,
307 SrcMem | ModRM | Stack, 0,
309 0, 0, ModRM | SrcMem | Priv, ModRM | SrcMem | Priv,
310 SrcNone | ModRM | DstMem | Mov, 0,
311 SrcMem16 | ModRM | Mov | Priv, SrcMem | ModRM | ByteOp | Priv,
314 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM | Lock,
315 DstMem | SrcImmByte | ModRM | Lock, DstMem | SrcImmByte | ModRM | Lock,
317 0, DstMem64 | ModRM | Lock, 0, 0, 0, 0, 0, 0,
320 static struct opcode group2_table[] = {
322 SrcNone | ModRM | Priv, 0, 0, SrcNone | ModRM | Priv,
323 SrcNone | ModRM | DstMem | Mov, 0,
324 SrcMem16 | ModRM | Mov | Priv, 0,
326 0, 0, 0, 0, 0, 0, 0, 0,
329 /* EFLAGS bit definitions. */
330 #define EFLG_ID (1<<21)
331 #define EFLG_VIP (1<<20)
332 #define EFLG_VIF (1<<19)
333 #define EFLG_AC (1<<18)
334 #define EFLG_VM (1<<17)
335 #define EFLG_RF (1<<16)
336 #define EFLG_IOPL (3<<12)
337 #define EFLG_NT (1<<14)
338 #define EFLG_OF (1<<11)
339 #define EFLG_DF (1<<10)
340 #define EFLG_IF (1<<9)
341 #define EFLG_TF (1<<8)
342 #define EFLG_SF (1<<7)
343 #define EFLG_ZF (1<<6)
344 #define EFLG_AF (1<<4)
345 #define EFLG_PF (1<<2)
346 #define EFLG_CF (1<<0)
348 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
349 #define EFLG_RESERVED_ONE_MASK 2
352 * Instruction emulation:
353 * Most instructions are emulated directly via a fragment of inline assembly
354 * code. This allows us to save/restore EFLAGS and thus very easily pick up
355 * any modified flags.
358 #if defined(CONFIG_X86_64)
359 #define _LO32 "k" /* force 32-bit operand */
360 #define _STK "%%rsp" /* stack pointer */
361 #elif defined(__i386__)
362 #define _LO32 "" /* force 32-bit operand */
363 #define _STK "%%esp" /* stack pointer */
367 * These EFLAGS bits are restored from saved value during emulation, and
368 * any changes are written back to the saved value after emulation.
370 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
372 /* Before executing instruction: restore necessary bits in EFLAGS. */
373 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
374 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
375 "movl %"_sav",%"_LO32 _tmp"; " \
378 "movl %"_msk",%"_LO32 _tmp"; " \
379 "andl %"_LO32 _tmp",("_STK"); " \
381 "notl %"_LO32 _tmp"; " \
382 "andl %"_LO32 _tmp",("_STK"); " \
383 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
385 "orl %"_LO32 _tmp",("_STK"); " \
389 /* After executing instruction: write-back necessary bits in EFLAGS. */
390 #define _POST_EFLAGS(_sav, _msk, _tmp) \
391 /* _sav |= EFLAGS & _msk; */ \
394 "andl %"_msk",%"_LO32 _tmp"; " \
395 "orl %"_LO32 _tmp",%"_sav"; "
403 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix) \
405 __asm__ __volatile__ ( \
406 _PRE_EFLAGS("0", "4", "2") \
407 _op _suffix " %"_x"3,%1; " \
408 _POST_EFLAGS("0", "4", "2") \
409 : "=m" (_eflags), "=m" ((_dst).val), \
411 : _y ((_src).val), "i" (EFLAGS_MASK)); \
415 /* Raw emulation: instruction has two explicit operands. */
416 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
418 unsigned long _tmp; \
420 switch ((_dst).bytes) { \
422 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w"); \
425 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l"); \
428 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q")); \
433 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
435 unsigned long _tmp; \
436 switch ((_dst).bytes) { \
438 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b"); \
441 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
442 _wx, _wy, _lx, _ly, _qx, _qy); \
447 /* Source operand is byte-sized and may be restricted to just %cl. */
448 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
449 __emulate_2op(_op, _src, _dst, _eflags, \
450 "b", "c", "b", "c", "b", "c", "b", "c")
452 /* Source operand is byte, word, long or quad sized. */
453 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
454 __emulate_2op(_op, _src, _dst, _eflags, \
455 "b", "q", "w", "r", _LO32, "r", "", "r")
457 /* Source operand is word, long or quad sized. */
458 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
459 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
460 "w", "r", _LO32, "r", "", "r")
462 /* Instruction has three operands and one operand is stored in ECX register */
463 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
465 unsigned long _tmp; \
466 _type _clv = (_cl).val; \
467 _type _srcv = (_src).val; \
468 _type _dstv = (_dst).val; \
470 __asm__ __volatile__ ( \
471 _PRE_EFLAGS("0", "5", "2") \
472 _op _suffix " %4,%1 \n" \
473 _POST_EFLAGS("0", "5", "2") \
474 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
475 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
478 (_cl).val = (unsigned long) _clv; \
479 (_src).val = (unsigned long) _srcv; \
480 (_dst).val = (unsigned long) _dstv; \
483 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
485 switch ((_dst).bytes) { \
487 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
488 "w", unsigned short); \
491 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
492 "l", unsigned int); \
495 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
496 "q", unsigned long)); \
501 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
503 unsigned long _tmp; \
505 __asm__ __volatile__ ( \
506 _PRE_EFLAGS("0", "3", "2") \
507 _op _suffix " %1; " \
508 _POST_EFLAGS("0", "3", "2") \
509 : "=m" (_eflags), "+m" ((_dst).val), \
511 : "i" (EFLAGS_MASK)); \
514 /* Instruction has only one explicit operand (no source operand). */
515 #define emulate_1op(_op, _dst, _eflags) \
517 switch ((_dst).bytes) { \
518 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
519 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
520 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
521 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
525 /* Fetch next part of the instruction being emulated. */
526 #define insn_fetch(_type, _size, _eip) \
527 ({ unsigned long _x; \
528 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
529 if (rc != X86EMUL_CONTINUE) \
535 #define insn_fetch_arr(_arr, _size, _eip) \
536 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
537 if (rc != X86EMUL_CONTINUE) \
542 static inline unsigned long ad_mask(struct decode_cache *c)
544 return (1UL << (c->ad_bytes << 3)) - 1;
547 /* Access/update address held in a register, based on addressing mode. */
548 static inline unsigned long
549 address_mask(struct decode_cache *c, unsigned long reg)
551 if (c->ad_bytes == sizeof(unsigned long))
554 return reg & ad_mask(c);
557 static inline unsigned long
558 register_address(struct decode_cache *c, unsigned long base, unsigned long reg)
560 return base + address_mask(c, reg);
564 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
566 if (c->ad_bytes == sizeof(unsigned long))
569 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
572 static inline void jmp_rel(struct decode_cache *c, int rel)
574 register_address_increment(c, &c->eip, rel);
577 static void set_seg_override(struct decode_cache *c, int seg)
579 c->has_seg_override = true;
580 c->seg_override = seg;
583 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
584 struct x86_emulate_ops *ops, int seg)
586 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
589 return ops->get_cached_segment_base(seg, ctxt->vcpu);
592 static unsigned long seg_override_base(struct x86_emulate_ctxt *ctxt,
593 struct x86_emulate_ops *ops,
594 struct decode_cache *c)
596 if (!c->has_seg_override)
599 return seg_base(ctxt, ops, c->seg_override);
602 static unsigned long es_base(struct x86_emulate_ctxt *ctxt,
603 struct x86_emulate_ops *ops)
605 return seg_base(ctxt, ops, VCPU_SREG_ES);
608 static unsigned long ss_base(struct x86_emulate_ctxt *ctxt,
609 struct x86_emulate_ops *ops)
611 return seg_base(ctxt, ops, VCPU_SREG_SS);
614 static void emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
615 u32 error, bool valid)
617 ctxt->exception = vec;
618 ctxt->error_code = error;
619 ctxt->error_code_valid = valid;
620 ctxt->restart = false;
623 static void emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
625 emulate_exception(ctxt, GP_VECTOR, err, true);
628 static void emulate_pf(struct x86_emulate_ctxt *ctxt, unsigned long addr,
632 emulate_exception(ctxt, PF_VECTOR, err, true);
635 static void emulate_ud(struct x86_emulate_ctxt *ctxt)
637 emulate_exception(ctxt, UD_VECTOR, 0, false);
640 static void emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
642 emulate_exception(ctxt, TS_VECTOR, err, true);
645 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
646 struct x86_emulate_ops *ops,
647 unsigned long eip, u8 *dest)
649 struct fetch_cache *fc = &ctxt->decode.fetch;
653 if (eip == fc->end) {
654 cur_size = fc->end - fc->start;
655 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
656 rc = ops->fetch(ctxt->cs_base + eip, fc->data + cur_size,
657 size, ctxt->vcpu, NULL);
658 if (rc != X86EMUL_CONTINUE)
662 *dest = fc->data[eip - fc->start];
663 return X86EMUL_CONTINUE;
666 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
667 struct x86_emulate_ops *ops,
668 unsigned long eip, void *dest, unsigned size)
672 /* x86 instructions are limited to 15 bytes. */
673 if (eip + size - ctxt->eip > 15)
674 return X86EMUL_UNHANDLEABLE;
676 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
677 if (rc != X86EMUL_CONTINUE)
680 return X86EMUL_CONTINUE;
684 * Given the 'reg' portion of a ModRM byte, and a register block, return a
685 * pointer into the block that addresses the relevant register.
686 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
688 static void *decode_register(u8 modrm_reg, unsigned long *regs,
693 p = ®s[modrm_reg];
694 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
695 p = (unsigned char *)®s[modrm_reg & 3] + 1;
699 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
700 struct x86_emulate_ops *ops,
702 u16 *size, unsigned long *address, int op_bytes)
709 rc = ops->read_std((unsigned long)ptr, (unsigned long *)size, 2,
711 if (rc != X86EMUL_CONTINUE)
713 rc = ops->read_std((unsigned long)ptr + 2, address, op_bytes,
718 static int test_cc(unsigned int condition, unsigned int flags)
722 switch ((condition & 15) >> 1) {
724 rc |= (flags & EFLG_OF);
726 case 1: /* b/c/nae */
727 rc |= (flags & EFLG_CF);
730 rc |= (flags & EFLG_ZF);
733 rc |= (flags & (EFLG_CF|EFLG_ZF));
736 rc |= (flags & EFLG_SF);
739 rc |= (flags & EFLG_PF);
742 rc |= (flags & EFLG_ZF);
745 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
749 /* Odd condition identifiers (lsb == 1) have inverted sense. */
750 return (!!rc ^ (condition & 1));
753 static void decode_register_operand(struct operand *op,
754 struct decode_cache *c,
757 unsigned reg = c->modrm_reg;
758 int highbyte_regs = c->rex_prefix == 0;
761 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
763 if ((c->d & ByteOp) && !inhibit_bytereg) {
764 op->ptr = decode_register(reg, c->regs, highbyte_regs);
765 op->val = *(u8 *)op->ptr;
768 op->ptr = decode_register(reg, c->regs, 0);
769 op->bytes = c->op_bytes;
772 op->val = *(u16 *)op->ptr;
775 op->val = *(u32 *)op->ptr;
778 op->val = *(u64 *) op->ptr;
782 op->orig_val = op->val;
785 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
786 struct x86_emulate_ops *ops)
788 struct decode_cache *c = &ctxt->decode;
790 int index_reg = 0, base_reg = 0, scale;
791 int rc = X86EMUL_CONTINUE;
794 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
795 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
796 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
799 c->modrm = insn_fetch(u8, 1, c->eip);
800 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
801 c->modrm_reg |= (c->modrm & 0x38) >> 3;
802 c->modrm_rm |= (c->modrm & 0x07);
806 if (c->modrm_mod == 3) {
807 c->modrm_ptr = decode_register(c->modrm_rm,
808 c->regs, c->d & ByteOp);
809 c->modrm_val = *(unsigned long *)c->modrm_ptr;
813 if (c->ad_bytes == 2) {
814 unsigned bx = c->regs[VCPU_REGS_RBX];
815 unsigned bp = c->regs[VCPU_REGS_RBP];
816 unsigned si = c->regs[VCPU_REGS_RSI];
817 unsigned di = c->regs[VCPU_REGS_RDI];
819 /* 16-bit ModR/M decode. */
820 switch (c->modrm_mod) {
822 if (c->modrm_rm == 6)
823 c->modrm_ea += insn_fetch(u16, 2, c->eip);
826 c->modrm_ea += insn_fetch(s8, 1, c->eip);
829 c->modrm_ea += insn_fetch(u16, 2, c->eip);
832 switch (c->modrm_rm) {
834 c->modrm_ea += bx + si;
837 c->modrm_ea += bx + di;
840 c->modrm_ea += bp + si;
843 c->modrm_ea += bp + di;
852 if (c->modrm_mod != 0)
859 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
860 (c->modrm_rm == 6 && c->modrm_mod != 0))
861 if (!c->has_seg_override)
862 set_seg_override(c, VCPU_SREG_SS);
863 c->modrm_ea = (u16)c->modrm_ea;
865 /* 32/64-bit ModR/M decode. */
866 if ((c->modrm_rm & 7) == 4) {
867 sib = insn_fetch(u8, 1, c->eip);
868 index_reg |= (sib >> 3) & 7;
872 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
873 c->modrm_ea += insn_fetch(s32, 4, c->eip);
875 c->modrm_ea += c->regs[base_reg];
877 c->modrm_ea += c->regs[index_reg] << scale;
878 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
879 if (ctxt->mode == X86EMUL_MODE_PROT64)
882 c->modrm_ea += c->regs[c->modrm_rm];
883 switch (c->modrm_mod) {
885 if (c->modrm_rm == 5)
886 c->modrm_ea += insn_fetch(s32, 4, c->eip);
889 c->modrm_ea += insn_fetch(s8, 1, c->eip);
892 c->modrm_ea += insn_fetch(s32, 4, c->eip);
900 static int decode_abs(struct x86_emulate_ctxt *ctxt,
901 struct x86_emulate_ops *ops)
903 struct decode_cache *c = &ctxt->decode;
904 int rc = X86EMUL_CONTINUE;
906 switch (c->ad_bytes) {
908 c->modrm_ea = insn_fetch(u16, 2, c->eip);
911 c->modrm_ea = insn_fetch(u32, 4, c->eip);
914 c->modrm_ea = insn_fetch(u64, 8, c->eip);
922 x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
924 struct decode_cache *c = &ctxt->decode;
925 int rc = X86EMUL_CONTINUE;
926 int mode = ctxt->mode;
927 int def_op_bytes, def_ad_bytes, group, dual;
930 /* we cannot decode insn before we complete previous rep insn */
931 WARN_ON(ctxt->restart);
934 c->fetch.start = c->fetch.end = c->eip;
935 ctxt->cs_base = seg_base(ctxt, ops, VCPU_SREG_CS);
938 case X86EMUL_MODE_REAL:
939 case X86EMUL_MODE_VM86:
940 case X86EMUL_MODE_PROT16:
941 def_op_bytes = def_ad_bytes = 2;
943 case X86EMUL_MODE_PROT32:
944 def_op_bytes = def_ad_bytes = 4;
947 case X86EMUL_MODE_PROT64:
956 c->op_bytes = def_op_bytes;
957 c->ad_bytes = def_ad_bytes;
959 /* Legacy prefixes. */
961 switch (c->b = insn_fetch(u8, 1, c->eip)) {
962 case 0x66: /* operand-size override */
963 /* switch between 2/4 bytes */
964 c->op_bytes = def_op_bytes ^ 6;
966 case 0x67: /* address-size override */
967 if (mode == X86EMUL_MODE_PROT64)
968 /* switch between 4/8 bytes */
969 c->ad_bytes = def_ad_bytes ^ 12;
971 /* switch between 2/4 bytes */
972 c->ad_bytes = def_ad_bytes ^ 6;
974 case 0x26: /* ES override */
975 case 0x2e: /* CS override */
976 case 0x36: /* SS override */
977 case 0x3e: /* DS override */
978 set_seg_override(c, (c->b >> 3) & 3);
980 case 0x64: /* FS override */
981 case 0x65: /* GS override */
982 set_seg_override(c, c->b & 7);
984 case 0x40 ... 0x4f: /* REX */
985 if (mode != X86EMUL_MODE_PROT64)
987 c->rex_prefix = c->b;
989 case 0xf0: /* LOCK */
992 case 0xf2: /* REPNE/REPNZ */
993 c->rep_prefix = REPNE_PREFIX;
995 case 0xf3: /* REP/REPE/REPZ */
996 c->rep_prefix = REPE_PREFIX;
1002 /* Any legacy prefix after a REX prefix nullifies its effect. */
1011 if (c->rex_prefix & 8)
1012 c->op_bytes = 8; /* REX.W */
1014 /* Opcode byte(s). */
1015 c->d = opcode_table[c->b].flags;
1017 /* Two-byte opcode? */
1020 c->b = insn_fetch(u8, 1, c->eip);
1021 c->d = twobyte_table[c->b].flags;
1026 group = c->d & GroupMask;
1027 dual = c->d & GroupDual;
1028 c->modrm = insn_fetch(u8, 1, c->eip);
1031 group = (group << 3) + ((c->modrm >> 3) & 7);
1032 c->d &= ~(Group | GroupDual | GroupMask);
1033 if (dual && (c->modrm >> 6) == 3)
1034 c->d |= group2_table[group].flags;
1036 c->d |= group_table[group].flags;
1040 if (c->d == 0 || (c->d & Undefined)) {
1041 DPRINTF("Cannot emulate %02x\n", c->b);
1045 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
1048 /* ModRM and SIB bytes. */
1050 rc = decode_modrm(ctxt, ops);
1051 else if (c->d & MemAbs)
1052 rc = decode_abs(ctxt, ops);
1053 if (rc != X86EMUL_CONTINUE)
1056 if (!c->has_seg_override)
1057 set_seg_override(c, VCPU_SREG_DS);
1059 if (!(!c->twobyte && c->b == 0x8d))
1060 c->modrm_ea += seg_override_base(ctxt, ops, c);
1062 if (c->ad_bytes != 8)
1063 c->modrm_ea = (u32)c->modrm_ea;
1065 if (c->rip_relative)
1066 c->modrm_ea += c->eip;
1069 * Decode and fetch the source operand: register, memory
1072 switch (c->d & SrcMask) {
1076 decode_register_operand(&c->src, c, 0);
1085 c->src.bytes = (c->d & ByteOp) ? 1 :
1087 /* Don't fetch the address for invlpg: it could be unmapped. */
1088 if (c->twobyte && c->b == 0x01 && c->modrm_reg == 7)
1092 * For instructions with a ModR/M byte, switch to register
1093 * access if Mod = 3.
1095 if ((c->d & ModRM) && c->modrm_mod == 3) {
1096 c->src.type = OP_REG;
1097 c->src.val = c->modrm_val;
1098 c->src.ptr = c->modrm_ptr;
1101 c->src.type = OP_MEM;
1102 c->src.ptr = (unsigned long *)c->modrm_ea;
1107 c->src.type = OP_IMM;
1108 c->src.ptr = (unsigned long *)c->eip;
1109 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1110 if (c->src.bytes == 8)
1112 /* NB. Immediates are sign-extended as necessary. */
1113 switch (c->src.bytes) {
1115 c->src.val = insn_fetch(s8, 1, c->eip);
1118 c->src.val = insn_fetch(s16, 2, c->eip);
1121 c->src.val = insn_fetch(s32, 4, c->eip);
1124 if ((c->d & SrcMask) == SrcImmU) {
1125 switch (c->src.bytes) {
1130 c->src.val &= 0xffff;
1133 c->src.val &= 0xffffffff;
1140 c->src.type = OP_IMM;
1141 c->src.ptr = (unsigned long *)c->eip;
1143 if ((c->d & SrcMask) == SrcImmByte)
1144 c->src.val = insn_fetch(s8, 1, c->eip);
1146 c->src.val = insn_fetch(u8, 1, c->eip);
1149 c->src.type = OP_REG;
1150 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1151 c->src.ptr = &c->regs[VCPU_REGS_RAX];
1152 switch (c->src.bytes) {
1154 c->src.val = *(u8 *)c->src.ptr;
1157 c->src.val = *(u16 *)c->src.ptr;
1160 c->src.val = *(u32 *)c->src.ptr;
1163 c->src.val = *(u64 *)c->src.ptr;
1172 c->src.type = OP_MEM;
1173 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1174 c->src.ptr = (unsigned long *)
1175 register_address(c, seg_override_base(ctxt, ops, c),
1176 c->regs[VCPU_REGS_RSI]);
1180 c->src.type = OP_IMM;
1181 c->src.ptr = (unsigned long *)c->eip;
1182 c->src.bytes = c->op_bytes + 2;
1183 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
1186 c->src.type = OP_MEM;
1187 c->src.ptr = (unsigned long *)c->modrm_ea;
1188 c->src.bytes = c->op_bytes + 2;
1193 * Decode and fetch the second source operand: register, memory
1196 switch (c->d & Src2Mask) {
1201 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
1204 c->src2.type = OP_IMM;
1205 c->src2.ptr = (unsigned long *)c->eip;
1207 c->src2.val = insn_fetch(u8, 1, c->eip);
1215 /* Decode and fetch the destination operand: register or memory. */
1216 switch (c->d & DstMask) {
1218 /* Special instructions do their own operand decoding. */
1221 decode_register_operand(&c->dst, c,
1222 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
1226 if ((c->d & ModRM) && c->modrm_mod == 3) {
1227 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1228 c->dst.type = OP_REG;
1229 c->dst.val = c->dst.orig_val = c->modrm_val;
1230 c->dst.ptr = c->modrm_ptr;
1233 c->dst.type = OP_MEM;
1234 c->dst.ptr = (unsigned long *)c->modrm_ea;
1235 if ((c->d & DstMask) == DstMem64)
1238 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1241 unsigned long mask = ~(c->dst.bytes * 8 - 1);
1243 c->dst.ptr = (void *)c->dst.ptr +
1244 (c->src.val & mask) / 8;
1248 c->dst.type = OP_REG;
1249 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1250 c->dst.ptr = &c->regs[VCPU_REGS_RAX];
1251 switch (c->dst.bytes) {
1253 c->dst.val = *(u8 *)c->dst.ptr;
1256 c->dst.val = *(u16 *)c->dst.ptr;
1259 c->dst.val = *(u32 *)c->dst.ptr;
1262 c->dst.val = *(u64 *)c->dst.ptr;
1265 c->dst.orig_val = c->dst.val;
1268 c->dst.type = OP_MEM;
1269 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1270 c->dst.ptr = (unsigned long *)
1271 register_address(c, es_base(ctxt, ops),
1272 c->regs[VCPU_REGS_RDI]);
1278 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
1281 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1282 struct x86_emulate_ops *ops,
1283 unsigned long addr, void *dest, unsigned size)
1286 struct read_cache *mc = &ctxt->decode.mem_read;
1290 int n = min(size, 8u);
1292 if (mc->pos < mc->end)
1295 rc = ops->read_emulated(addr, mc->data + mc->end, n, &err,
1297 if (rc == X86EMUL_PROPAGATE_FAULT)
1298 emulate_pf(ctxt, addr, err);
1299 if (rc != X86EMUL_CONTINUE)
1304 memcpy(dest, mc->data + mc->pos, n);
1309 return X86EMUL_CONTINUE;
1312 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1313 struct x86_emulate_ops *ops,
1314 unsigned int size, unsigned short port,
1317 struct read_cache *rc = &ctxt->decode.io_read;
1319 if (rc->pos == rc->end) { /* refill pio read ahead */
1320 struct decode_cache *c = &ctxt->decode;
1321 unsigned int in_page, n;
1322 unsigned int count = c->rep_prefix ?
1323 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
1324 in_page = (ctxt->eflags & EFLG_DF) ?
1325 offset_in_page(c->regs[VCPU_REGS_RDI]) :
1326 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
1327 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1331 rc->pos = rc->end = 0;
1332 if (!ops->pio_in_emulated(size, port, rc->data, n, ctxt->vcpu))
1337 memcpy(dest, rc->data + rc->pos, size);
1342 static u32 desc_limit_scaled(struct desc_struct *desc)
1344 u32 limit = get_desc_limit(desc);
1346 return desc->g ? (limit << 12) | 0xfff : limit;
1349 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1350 struct x86_emulate_ops *ops,
1351 u16 selector, struct desc_ptr *dt)
1353 if (selector & 1 << 2) {
1354 struct desc_struct desc;
1355 memset (dt, 0, sizeof *dt);
1356 if (!ops->get_cached_descriptor(&desc, VCPU_SREG_LDTR, ctxt->vcpu))
1359 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1360 dt->address = get_desc_base(&desc);
1362 ops->get_gdt(dt, ctxt->vcpu);
1365 /* allowed just for 8 bytes segments */
1366 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1367 struct x86_emulate_ops *ops,
1368 u16 selector, struct desc_struct *desc)
1371 u16 index = selector >> 3;
1376 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1378 if (dt.size < index * 8 + 7) {
1379 emulate_gp(ctxt, selector & 0xfffc);
1380 return X86EMUL_PROPAGATE_FAULT;
1382 addr = dt.address + index * 8;
1383 ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu, &err);
1384 if (ret == X86EMUL_PROPAGATE_FAULT)
1385 emulate_pf(ctxt, addr, err);
1390 /* allowed just for 8 bytes segments */
1391 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1392 struct x86_emulate_ops *ops,
1393 u16 selector, struct desc_struct *desc)
1396 u16 index = selector >> 3;
1401 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1403 if (dt.size < index * 8 + 7) {
1404 emulate_gp(ctxt, selector & 0xfffc);
1405 return X86EMUL_PROPAGATE_FAULT;
1408 addr = dt.address + index * 8;
1409 ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu, &err);
1410 if (ret == X86EMUL_PROPAGATE_FAULT)
1411 emulate_pf(ctxt, addr, err);
1416 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1417 struct x86_emulate_ops *ops,
1418 u16 selector, int seg)
1420 struct desc_struct seg_desc;
1422 unsigned err_vec = GP_VECTOR;
1424 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1427 memset(&seg_desc, 0, sizeof seg_desc);
1429 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1430 || ctxt->mode == X86EMUL_MODE_REAL) {
1431 /* set real mode segment descriptor */
1432 set_desc_base(&seg_desc, selector << 4);
1433 set_desc_limit(&seg_desc, 0xffff);
1440 /* NULL selector is not valid for TR, CS and SS */
1441 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1445 /* TR should be in GDT only */
1446 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1449 if (null_selector) /* for NULL selector skip all following checks */
1452 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1453 if (ret != X86EMUL_CONTINUE)
1456 err_code = selector & 0xfffc;
1457 err_vec = GP_VECTOR;
1459 /* can't load system descriptor into segment selecor */
1460 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1464 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1470 cpl = ops->cpl(ctxt->vcpu);
1475 * segment is not a writable data segment or segment
1476 * selector's RPL != CPL or segment selector's RPL != CPL
1478 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1482 if (!(seg_desc.type & 8))
1485 if (seg_desc.type & 4) {
1491 if (rpl > cpl || dpl != cpl)
1494 /* CS(RPL) <- CPL */
1495 selector = (selector & 0xfffc) | cpl;
1498 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1501 case VCPU_SREG_LDTR:
1502 if (seg_desc.s || seg_desc.type != 2)
1505 default: /* DS, ES, FS, or GS */
1507 * segment is not a data or readable code segment or
1508 * ((segment is a data or nonconforming code segment)
1509 * and (both RPL and CPL > DPL))
1511 if ((seg_desc.type & 0xa) == 0x8 ||
1512 (((seg_desc.type & 0xc) != 0xc) &&
1513 (rpl > dpl && cpl > dpl)))
1519 /* mark segment as accessed */
1521 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1522 if (ret != X86EMUL_CONTINUE)
1526 ops->set_segment_selector(selector, seg, ctxt->vcpu);
1527 ops->set_cached_descriptor(&seg_desc, seg, ctxt->vcpu);
1528 return X86EMUL_CONTINUE;
1530 emulate_exception(ctxt, err_vec, err_code, true);
1531 return X86EMUL_PROPAGATE_FAULT;
1534 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1535 struct x86_emulate_ops *ops)
1538 struct decode_cache *c = &ctxt->decode;
1541 switch (c->dst.type) {
1543 /* The 4-byte case *is* correct:
1544 * in 64-bit mode we zero-extend.
1546 switch (c->dst.bytes) {
1548 *(u8 *)c->dst.ptr = (u8)c->dst.val;
1551 *(u16 *)c->dst.ptr = (u16)c->dst.val;
1554 *c->dst.ptr = (u32)c->dst.val;
1555 break; /* 64b: zero-ext */
1557 *c->dst.ptr = c->dst.val;
1563 rc = ops->cmpxchg_emulated(
1564 (unsigned long)c->dst.ptr,
1571 rc = ops->write_emulated(
1572 (unsigned long)c->dst.ptr,
1577 if (rc == X86EMUL_PROPAGATE_FAULT)
1579 (unsigned long)c->dst.ptr, err);
1580 if (rc != X86EMUL_CONTINUE)
1589 return X86EMUL_CONTINUE;
1592 static inline void emulate_push(struct x86_emulate_ctxt *ctxt,
1593 struct x86_emulate_ops *ops)
1595 struct decode_cache *c = &ctxt->decode;
1597 c->dst.type = OP_MEM;
1598 c->dst.bytes = c->op_bytes;
1599 c->dst.val = c->src.val;
1600 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1601 c->dst.ptr = (void *) register_address(c, ss_base(ctxt, ops),
1602 c->regs[VCPU_REGS_RSP]);
1605 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1606 struct x86_emulate_ops *ops,
1607 void *dest, int len)
1609 struct decode_cache *c = &ctxt->decode;
1612 rc = read_emulated(ctxt, ops, register_address(c, ss_base(ctxt, ops),
1613 c->regs[VCPU_REGS_RSP]),
1615 if (rc != X86EMUL_CONTINUE)
1618 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1622 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1623 struct x86_emulate_ops *ops,
1624 void *dest, int len)
1627 unsigned long val, change_mask;
1628 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1629 int cpl = ops->cpl(ctxt->vcpu);
1631 rc = emulate_pop(ctxt, ops, &val, len);
1632 if (rc != X86EMUL_CONTINUE)
1635 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1636 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1638 switch(ctxt->mode) {
1639 case X86EMUL_MODE_PROT64:
1640 case X86EMUL_MODE_PROT32:
1641 case X86EMUL_MODE_PROT16:
1643 change_mask |= EFLG_IOPL;
1645 change_mask |= EFLG_IF;
1647 case X86EMUL_MODE_VM86:
1649 emulate_gp(ctxt, 0);
1650 return X86EMUL_PROPAGATE_FAULT;
1652 change_mask |= EFLG_IF;
1654 default: /* real mode */
1655 change_mask |= (EFLG_IOPL | EFLG_IF);
1659 *(unsigned long *)dest =
1660 (ctxt->eflags & ~change_mask) | (val & change_mask);
1665 static void emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1666 struct x86_emulate_ops *ops, int seg)
1668 struct decode_cache *c = &ctxt->decode;
1670 c->src.val = ops->get_segment_selector(seg, ctxt->vcpu);
1672 emulate_push(ctxt, ops);
1675 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1676 struct x86_emulate_ops *ops, int seg)
1678 struct decode_cache *c = &ctxt->decode;
1679 unsigned long selector;
1682 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1683 if (rc != X86EMUL_CONTINUE)
1686 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1690 static int emulate_pusha(struct x86_emulate_ctxt *ctxt,
1691 struct x86_emulate_ops *ops)
1693 struct decode_cache *c = &ctxt->decode;
1694 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1695 int rc = X86EMUL_CONTINUE;
1696 int reg = VCPU_REGS_RAX;
1698 while (reg <= VCPU_REGS_RDI) {
1699 (reg == VCPU_REGS_RSP) ?
1700 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1702 emulate_push(ctxt, ops);
1704 rc = writeback(ctxt, ops);
1705 if (rc != X86EMUL_CONTINUE)
1711 /* Disable writeback. */
1712 c->dst.type = OP_NONE;
1717 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1718 struct x86_emulate_ops *ops)
1720 struct decode_cache *c = &ctxt->decode;
1721 int rc = X86EMUL_CONTINUE;
1722 int reg = VCPU_REGS_RDI;
1724 while (reg >= VCPU_REGS_RAX) {
1725 if (reg == VCPU_REGS_RSP) {
1726 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1731 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1732 if (rc != X86EMUL_CONTINUE)
1739 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
1740 struct x86_emulate_ops *ops)
1742 struct decode_cache *c = &ctxt->decode;
1743 int rc = X86EMUL_CONTINUE;
1744 unsigned long temp_eip = 0;
1745 unsigned long temp_eflags = 0;
1746 unsigned long cs = 0;
1747 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1748 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1749 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1750 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1752 /* TODO: Add stack limit check */
1754 rc = emulate_pop(ctxt, ops, &temp_eip, c->op_bytes);
1756 if (rc != X86EMUL_CONTINUE)
1759 if (temp_eip & ~0xffff) {
1760 emulate_gp(ctxt, 0);
1761 return X86EMUL_PROPAGATE_FAULT;
1764 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1766 if (rc != X86EMUL_CONTINUE)
1769 rc = emulate_pop(ctxt, ops, &temp_eflags, c->op_bytes);
1771 if (rc != X86EMUL_CONTINUE)
1774 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1776 if (rc != X86EMUL_CONTINUE)
1782 if (c->op_bytes == 4)
1783 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1784 else if (c->op_bytes == 2) {
1785 ctxt->eflags &= ~0xffff;
1786 ctxt->eflags |= temp_eflags;
1789 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1790 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1795 static inline int emulate_iret(struct x86_emulate_ctxt *ctxt,
1796 struct x86_emulate_ops* ops)
1798 switch(ctxt->mode) {
1799 case X86EMUL_MODE_REAL:
1800 return emulate_iret_real(ctxt, ops);
1801 case X86EMUL_MODE_VM86:
1802 case X86EMUL_MODE_PROT16:
1803 case X86EMUL_MODE_PROT32:
1804 case X86EMUL_MODE_PROT64:
1806 /* iret from protected mode unimplemented yet */
1807 return X86EMUL_UNHANDLEABLE;
1811 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1812 struct x86_emulate_ops *ops)
1814 struct decode_cache *c = &ctxt->decode;
1816 return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1819 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1821 struct decode_cache *c = &ctxt->decode;
1822 switch (c->modrm_reg) {
1824 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1827 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1830 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1833 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1835 case 4: /* sal/shl */
1836 case 6: /* sal/shl */
1837 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1840 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1843 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1848 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1849 struct x86_emulate_ops *ops)
1851 struct decode_cache *c = &ctxt->decode;
1853 switch (c->modrm_reg) {
1854 case 0 ... 1: /* test */
1855 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1858 c->dst.val = ~c->dst.val;
1861 emulate_1op("neg", c->dst, ctxt->eflags);
1869 static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
1870 struct x86_emulate_ops *ops)
1872 struct decode_cache *c = &ctxt->decode;
1874 switch (c->modrm_reg) {
1876 emulate_1op("inc", c->dst, ctxt->eflags);
1879 emulate_1op("dec", c->dst, ctxt->eflags);
1881 case 2: /* call near abs */ {
1884 c->eip = c->src.val;
1885 c->src.val = old_eip;
1886 emulate_push(ctxt, ops);
1889 case 4: /* jmp abs */
1890 c->eip = c->src.val;
1893 emulate_push(ctxt, ops);
1896 return X86EMUL_CONTINUE;
1899 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1900 struct x86_emulate_ops *ops)
1902 struct decode_cache *c = &ctxt->decode;
1903 u64 old = c->dst.orig_val64;
1905 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1906 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1907 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1908 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1909 ctxt->eflags &= ~EFLG_ZF;
1911 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1912 (u32) c->regs[VCPU_REGS_RBX];
1914 ctxt->eflags |= EFLG_ZF;
1916 return X86EMUL_CONTINUE;
1919 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1920 struct x86_emulate_ops *ops)
1922 struct decode_cache *c = &ctxt->decode;
1926 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1927 if (rc != X86EMUL_CONTINUE)
1929 if (c->op_bytes == 4)
1930 c->eip = (u32)c->eip;
1931 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1932 if (rc != X86EMUL_CONTINUE)
1934 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1939 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1940 struct x86_emulate_ops *ops, struct desc_struct *cs,
1941 struct desc_struct *ss)
1943 memset(cs, 0, sizeof(struct desc_struct));
1944 ops->get_cached_descriptor(cs, VCPU_SREG_CS, ctxt->vcpu);
1945 memset(ss, 0, sizeof(struct desc_struct));
1947 cs->l = 0; /* will be adjusted later */
1948 set_desc_base(cs, 0); /* flat segment */
1949 cs->g = 1; /* 4kb granularity */
1950 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1951 cs->type = 0x0b; /* Read, Execute, Accessed */
1953 cs->dpl = 0; /* will be adjusted later */
1957 set_desc_base(ss, 0); /* flat segment */
1958 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1959 ss->g = 1; /* 4kb granularity */
1961 ss->type = 0x03; /* Read/Write, Accessed */
1962 ss->d = 1; /* 32bit stack segment */
1968 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1970 struct decode_cache *c = &ctxt->decode;
1971 struct desc_struct cs, ss;
1975 /* syscall is not available in real mode */
1976 if (ctxt->mode == X86EMUL_MODE_REAL ||
1977 ctxt->mode == X86EMUL_MODE_VM86) {
1979 return X86EMUL_PROPAGATE_FAULT;
1982 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1984 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1986 cs_sel = (u16)(msr_data & 0xfffc);
1987 ss_sel = (u16)(msr_data + 8);
1989 if (is_long_mode(ctxt->vcpu)) {
1993 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
1994 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1995 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
1996 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1998 c->regs[VCPU_REGS_RCX] = c->eip;
1999 if (is_long_mode(ctxt->vcpu)) {
2000 #ifdef CONFIG_X86_64
2001 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
2003 ops->get_msr(ctxt->vcpu,
2004 ctxt->mode == X86EMUL_MODE_PROT64 ?
2005 MSR_LSTAR : MSR_CSTAR, &msr_data);
2008 ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
2009 ctxt->eflags &= ~(msr_data | EFLG_RF);
2013 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
2014 c->eip = (u32)msr_data;
2016 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2019 return X86EMUL_CONTINUE;
2023 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2025 struct decode_cache *c = &ctxt->decode;
2026 struct desc_struct cs, ss;
2030 /* inject #GP if in real mode */
2031 if (ctxt->mode == X86EMUL_MODE_REAL) {
2032 emulate_gp(ctxt, 0);
2033 return X86EMUL_PROPAGATE_FAULT;
2036 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2037 * Therefore, we inject an #UD.
2039 if (ctxt->mode == X86EMUL_MODE_PROT64) {
2041 return X86EMUL_PROPAGATE_FAULT;
2044 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2046 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
2047 switch (ctxt->mode) {
2048 case X86EMUL_MODE_PROT32:
2049 if ((msr_data & 0xfffc) == 0x0) {
2050 emulate_gp(ctxt, 0);
2051 return X86EMUL_PROPAGATE_FAULT;
2054 case X86EMUL_MODE_PROT64:
2055 if (msr_data == 0x0) {
2056 emulate_gp(ctxt, 0);
2057 return X86EMUL_PROPAGATE_FAULT;
2062 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2063 cs_sel = (u16)msr_data;
2064 cs_sel &= ~SELECTOR_RPL_MASK;
2065 ss_sel = cs_sel + 8;
2066 ss_sel &= ~SELECTOR_RPL_MASK;
2067 if (ctxt->mode == X86EMUL_MODE_PROT64
2068 || is_long_mode(ctxt->vcpu)) {
2073 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
2074 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
2075 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
2076 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
2078 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
2081 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
2082 c->regs[VCPU_REGS_RSP] = msr_data;
2084 return X86EMUL_CONTINUE;
2088 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2090 struct decode_cache *c = &ctxt->decode;
2091 struct desc_struct cs, ss;
2096 /* inject #GP if in real mode or Virtual 8086 mode */
2097 if (ctxt->mode == X86EMUL_MODE_REAL ||
2098 ctxt->mode == X86EMUL_MODE_VM86) {
2099 emulate_gp(ctxt, 0);
2100 return X86EMUL_PROPAGATE_FAULT;
2103 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2105 if ((c->rex_prefix & 0x8) != 0x0)
2106 usermode = X86EMUL_MODE_PROT64;
2108 usermode = X86EMUL_MODE_PROT32;
2112 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
2114 case X86EMUL_MODE_PROT32:
2115 cs_sel = (u16)(msr_data + 16);
2116 if ((msr_data & 0xfffc) == 0x0) {
2117 emulate_gp(ctxt, 0);
2118 return X86EMUL_PROPAGATE_FAULT;
2120 ss_sel = (u16)(msr_data + 24);
2122 case X86EMUL_MODE_PROT64:
2123 cs_sel = (u16)(msr_data + 32);
2124 if (msr_data == 0x0) {
2125 emulate_gp(ctxt, 0);
2126 return X86EMUL_PROPAGATE_FAULT;
2128 ss_sel = cs_sel + 8;
2133 cs_sel |= SELECTOR_RPL_MASK;
2134 ss_sel |= SELECTOR_RPL_MASK;
2136 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
2137 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
2138 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
2139 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
2141 c->eip = c->regs[VCPU_REGS_RDX];
2142 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
2144 return X86EMUL_CONTINUE;
2147 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
2148 struct x86_emulate_ops *ops)
2151 if (ctxt->mode == X86EMUL_MODE_REAL)
2153 if (ctxt->mode == X86EMUL_MODE_VM86)
2155 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2156 return ops->cpl(ctxt->vcpu) > iopl;
2159 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2160 struct x86_emulate_ops *ops,
2163 struct desc_struct tr_seg;
2166 u8 perm, bit_idx = port & 0x7;
2167 unsigned mask = (1 << len) - 1;
2169 ops->get_cached_descriptor(&tr_seg, VCPU_SREG_TR, ctxt->vcpu);
2172 if (desc_limit_scaled(&tr_seg) < 103)
2174 r = ops->read_std(get_desc_base(&tr_seg) + 102, &io_bitmap_ptr, 2,
2176 if (r != X86EMUL_CONTINUE)
2178 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2180 r = ops->read_std(get_desc_base(&tr_seg) + io_bitmap_ptr + port/8,
2181 &perm, 1, ctxt->vcpu, NULL);
2182 if (r != X86EMUL_CONTINUE)
2184 if ((perm >> bit_idx) & mask)
2189 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2190 struct x86_emulate_ops *ops,
2193 if (emulator_bad_iopl(ctxt, ops))
2194 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
2199 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2200 struct x86_emulate_ops *ops,
2201 struct tss_segment_16 *tss)
2203 struct decode_cache *c = &ctxt->decode;
2206 tss->flag = ctxt->eflags;
2207 tss->ax = c->regs[VCPU_REGS_RAX];
2208 tss->cx = c->regs[VCPU_REGS_RCX];
2209 tss->dx = c->regs[VCPU_REGS_RDX];
2210 tss->bx = c->regs[VCPU_REGS_RBX];
2211 tss->sp = c->regs[VCPU_REGS_RSP];
2212 tss->bp = c->regs[VCPU_REGS_RBP];
2213 tss->si = c->regs[VCPU_REGS_RSI];
2214 tss->di = c->regs[VCPU_REGS_RDI];
2216 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2217 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2218 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2219 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2220 tss->ldt = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2223 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2224 struct x86_emulate_ops *ops,
2225 struct tss_segment_16 *tss)
2227 struct decode_cache *c = &ctxt->decode;
2231 ctxt->eflags = tss->flag | 2;
2232 c->regs[VCPU_REGS_RAX] = tss->ax;
2233 c->regs[VCPU_REGS_RCX] = tss->cx;
2234 c->regs[VCPU_REGS_RDX] = tss->dx;
2235 c->regs[VCPU_REGS_RBX] = tss->bx;
2236 c->regs[VCPU_REGS_RSP] = tss->sp;
2237 c->regs[VCPU_REGS_RBP] = tss->bp;
2238 c->regs[VCPU_REGS_RSI] = tss->si;
2239 c->regs[VCPU_REGS_RDI] = tss->di;
2242 * SDM says that segment selectors are loaded before segment
2245 ops->set_segment_selector(tss->ldt, VCPU_SREG_LDTR, ctxt->vcpu);
2246 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2247 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2248 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2249 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2252 * Now load segment descriptors. If fault happenes at this stage
2253 * it is handled in a context of new task
2255 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
2256 if (ret != X86EMUL_CONTINUE)
2258 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2259 if (ret != X86EMUL_CONTINUE)
2261 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2262 if (ret != X86EMUL_CONTINUE)
2264 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2265 if (ret != X86EMUL_CONTINUE)
2267 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2268 if (ret != X86EMUL_CONTINUE)
2271 return X86EMUL_CONTINUE;
2274 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2275 struct x86_emulate_ops *ops,
2276 u16 tss_selector, u16 old_tss_sel,
2277 ulong old_tss_base, struct desc_struct *new_desc)
2279 struct tss_segment_16 tss_seg;
2281 u32 err, new_tss_base = get_desc_base(new_desc);
2283 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2285 if (ret == X86EMUL_PROPAGATE_FAULT) {
2286 /* FIXME: need to provide precise fault address */
2287 emulate_pf(ctxt, old_tss_base, err);
2291 save_state_to_tss16(ctxt, ops, &tss_seg);
2293 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2295 if (ret == X86EMUL_PROPAGATE_FAULT) {
2296 /* FIXME: need to provide precise fault address */
2297 emulate_pf(ctxt, old_tss_base, err);
2301 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2303 if (ret == X86EMUL_PROPAGATE_FAULT) {
2304 /* FIXME: need to provide precise fault address */
2305 emulate_pf(ctxt, new_tss_base, err);
2309 if (old_tss_sel != 0xffff) {
2310 tss_seg.prev_task_link = old_tss_sel;
2312 ret = ops->write_std(new_tss_base,
2313 &tss_seg.prev_task_link,
2314 sizeof tss_seg.prev_task_link,
2316 if (ret == X86EMUL_PROPAGATE_FAULT) {
2317 /* FIXME: need to provide precise fault address */
2318 emulate_pf(ctxt, new_tss_base, err);
2323 return load_state_from_tss16(ctxt, ops, &tss_seg);
2326 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2327 struct x86_emulate_ops *ops,
2328 struct tss_segment_32 *tss)
2330 struct decode_cache *c = &ctxt->decode;
2332 tss->cr3 = ops->get_cr(3, ctxt->vcpu);
2334 tss->eflags = ctxt->eflags;
2335 tss->eax = c->regs[VCPU_REGS_RAX];
2336 tss->ecx = c->regs[VCPU_REGS_RCX];
2337 tss->edx = c->regs[VCPU_REGS_RDX];
2338 tss->ebx = c->regs[VCPU_REGS_RBX];
2339 tss->esp = c->regs[VCPU_REGS_RSP];
2340 tss->ebp = c->regs[VCPU_REGS_RBP];
2341 tss->esi = c->regs[VCPU_REGS_RSI];
2342 tss->edi = c->regs[VCPU_REGS_RDI];
2344 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2345 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2346 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2347 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2348 tss->fs = ops->get_segment_selector(VCPU_SREG_FS, ctxt->vcpu);
2349 tss->gs = ops->get_segment_selector(VCPU_SREG_GS, ctxt->vcpu);
2350 tss->ldt_selector = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2353 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2354 struct x86_emulate_ops *ops,
2355 struct tss_segment_32 *tss)
2357 struct decode_cache *c = &ctxt->decode;
2360 if (ops->set_cr(3, tss->cr3, ctxt->vcpu)) {
2361 emulate_gp(ctxt, 0);
2362 return X86EMUL_PROPAGATE_FAULT;
2365 ctxt->eflags = tss->eflags | 2;
2366 c->regs[VCPU_REGS_RAX] = tss->eax;
2367 c->regs[VCPU_REGS_RCX] = tss->ecx;
2368 c->regs[VCPU_REGS_RDX] = tss->edx;
2369 c->regs[VCPU_REGS_RBX] = tss->ebx;
2370 c->regs[VCPU_REGS_RSP] = tss->esp;
2371 c->regs[VCPU_REGS_RBP] = tss->ebp;
2372 c->regs[VCPU_REGS_RSI] = tss->esi;
2373 c->regs[VCPU_REGS_RDI] = tss->edi;
2376 * SDM says that segment selectors are loaded before segment
2379 ops->set_segment_selector(tss->ldt_selector, VCPU_SREG_LDTR, ctxt->vcpu);
2380 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2381 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2382 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2383 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2384 ops->set_segment_selector(tss->fs, VCPU_SREG_FS, ctxt->vcpu);
2385 ops->set_segment_selector(tss->gs, VCPU_SREG_GS, ctxt->vcpu);
2388 * Now load segment descriptors. If fault happenes at this stage
2389 * it is handled in a context of new task
2391 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2392 if (ret != X86EMUL_CONTINUE)
2394 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2395 if (ret != X86EMUL_CONTINUE)
2397 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2398 if (ret != X86EMUL_CONTINUE)
2400 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2401 if (ret != X86EMUL_CONTINUE)
2403 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2404 if (ret != X86EMUL_CONTINUE)
2406 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2407 if (ret != X86EMUL_CONTINUE)
2409 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2410 if (ret != X86EMUL_CONTINUE)
2413 return X86EMUL_CONTINUE;
2416 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2417 struct x86_emulate_ops *ops,
2418 u16 tss_selector, u16 old_tss_sel,
2419 ulong old_tss_base, struct desc_struct *new_desc)
2421 struct tss_segment_32 tss_seg;
2423 u32 err, new_tss_base = get_desc_base(new_desc);
2425 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2427 if (ret == X86EMUL_PROPAGATE_FAULT) {
2428 /* FIXME: need to provide precise fault address */
2429 emulate_pf(ctxt, old_tss_base, err);
2433 save_state_to_tss32(ctxt, ops, &tss_seg);
2435 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2437 if (ret == X86EMUL_PROPAGATE_FAULT) {
2438 /* FIXME: need to provide precise fault address */
2439 emulate_pf(ctxt, old_tss_base, err);
2443 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2445 if (ret == X86EMUL_PROPAGATE_FAULT) {
2446 /* FIXME: need to provide precise fault address */
2447 emulate_pf(ctxt, new_tss_base, err);
2451 if (old_tss_sel != 0xffff) {
2452 tss_seg.prev_task_link = old_tss_sel;
2454 ret = ops->write_std(new_tss_base,
2455 &tss_seg.prev_task_link,
2456 sizeof tss_seg.prev_task_link,
2458 if (ret == X86EMUL_PROPAGATE_FAULT) {
2459 /* FIXME: need to provide precise fault address */
2460 emulate_pf(ctxt, new_tss_base, err);
2465 return load_state_from_tss32(ctxt, ops, &tss_seg);
2468 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2469 struct x86_emulate_ops *ops,
2470 u16 tss_selector, int reason,
2471 bool has_error_code, u32 error_code)
2473 struct desc_struct curr_tss_desc, next_tss_desc;
2475 u16 old_tss_sel = ops->get_segment_selector(VCPU_SREG_TR, ctxt->vcpu);
2476 ulong old_tss_base =
2477 ops->get_cached_segment_base(VCPU_SREG_TR, ctxt->vcpu);
2480 /* FIXME: old_tss_base == ~0 ? */
2482 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2483 if (ret != X86EMUL_CONTINUE)
2485 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2486 if (ret != X86EMUL_CONTINUE)
2489 /* FIXME: check that next_tss_desc is tss */
2491 if (reason != TASK_SWITCH_IRET) {
2492 if ((tss_selector & 3) > next_tss_desc.dpl ||
2493 ops->cpl(ctxt->vcpu) > next_tss_desc.dpl) {
2494 emulate_gp(ctxt, 0);
2495 return X86EMUL_PROPAGATE_FAULT;
2499 desc_limit = desc_limit_scaled(&next_tss_desc);
2500 if (!next_tss_desc.p ||
2501 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2502 desc_limit < 0x2b)) {
2503 emulate_ts(ctxt, tss_selector & 0xfffc);
2504 return X86EMUL_PROPAGATE_FAULT;
2507 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2508 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2509 write_segment_descriptor(ctxt, ops, old_tss_sel,
2513 if (reason == TASK_SWITCH_IRET)
2514 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2516 /* set back link to prev task only if NT bit is set in eflags
2517 note that old_tss_sel is not used afetr this point */
2518 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2519 old_tss_sel = 0xffff;
2521 if (next_tss_desc.type & 8)
2522 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2523 old_tss_base, &next_tss_desc);
2525 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2526 old_tss_base, &next_tss_desc);
2527 if (ret != X86EMUL_CONTINUE)
2530 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2531 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2533 if (reason != TASK_SWITCH_IRET) {
2534 next_tss_desc.type |= (1 << 1); /* set busy flag */
2535 write_segment_descriptor(ctxt, ops, tss_selector,
2539 ops->set_cr(0, ops->get_cr(0, ctxt->vcpu) | X86_CR0_TS, ctxt->vcpu);
2540 ops->set_cached_descriptor(&next_tss_desc, VCPU_SREG_TR, ctxt->vcpu);
2541 ops->set_segment_selector(tss_selector, VCPU_SREG_TR, ctxt->vcpu);
2543 if (has_error_code) {
2544 struct decode_cache *c = &ctxt->decode;
2546 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2548 c->src.val = (unsigned long) error_code;
2549 emulate_push(ctxt, ops);
2555 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2556 struct x86_emulate_ops *ops,
2557 u16 tss_selector, int reason,
2558 bool has_error_code, u32 error_code)
2560 struct decode_cache *c = &ctxt->decode;
2564 c->dst.type = OP_NONE;
2566 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2567 has_error_code, error_code);
2569 if (rc == X86EMUL_CONTINUE) {
2570 rc = writeback(ctxt, ops);
2571 if (rc == X86EMUL_CONTINUE)
2575 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
2578 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned long base,
2579 int reg, struct operand *op)
2581 struct decode_cache *c = &ctxt->decode;
2582 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2584 register_address_increment(c, &c->regs[reg], df * op->bytes);
2585 op->ptr = (unsigned long *)register_address(c, base, c->regs[reg]);
2589 x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2592 struct decode_cache *c = &ctxt->decode;
2593 int rc = X86EMUL_CONTINUE;
2594 int saved_dst_type = c->dst.type;
2596 ctxt->decode.mem_read.pos = 0;
2598 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
2603 /* LOCK prefix is allowed only with some instructions */
2604 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
2609 /* Privileged instruction can be executed only in CPL=0 */
2610 if ((c->d & Priv) && ops->cpl(ctxt->vcpu)) {
2611 emulate_gp(ctxt, 0);
2615 if (c->rep_prefix && (c->d & String)) {
2616 ctxt->restart = true;
2617 /* All REP prefixes have the same first termination condition */
2618 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
2620 ctxt->restart = false;
2624 /* The second termination condition only applies for REPE
2625 * and REPNE. Test if the repeat string operation prefix is
2626 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
2627 * corresponding termination condition according to:
2628 * - if REPE/REPZ and ZF = 0 then done
2629 * - if REPNE/REPNZ and ZF = 1 then done
2631 if ((c->b == 0xa6) || (c->b == 0xa7) ||
2632 (c->b == 0xae) || (c->b == 0xaf)) {
2633 if ((c->rep_prefix == REPE_PREFIX) &&
2634 ((ctxt->eflags & EFLG_ZF) == 0))
2636 if ((c->rep_prefix == REPNE_PREFIX) &&
2637 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))
2643 if (c->src.type == OP_MEM) {
2644 rc = read_emulated(ctxt, ops, (unsigned long)c->src.ptr,
2645 c->src.valptr, c->src.bytes);
2646 if (rc != X86EMUL_CONTINUE)
2648 c->src.orig_val64 = c->src.val64;
2651 if (c->src2.type == OP_MEM) {
2652 rc = read_emulated(ctxt, ops, (unsigned long)c->src2.ptr,
2653 &c->src2.val, c->src2.bytes);
2654 if (rc != X86EMUL_CONTINUE)
2658 if ((c->d & DstMask) == ImplicitOps)
2662 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
2663 /* optimisation - avoid slow emulated read if Mov */
2664 rc = read_emulated(ctxt, ops, (unsigned long)c->dst.ptr,
2665 &c->dst.val, c->dst.bytes);
2666 if (rc != X86EMUL_CONTINUE)
2669 c->dst.orig_val = c->dst.val;
2679 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
2681 case 0x06: /* push es */
2682 emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
2684 case 0x07: /* pop es */
2685 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
2686 if (rc != X86EMUL_CONTINUE)
2691 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2693 case 0x0e: /* push cs */
2694 emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
2698 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
2700 case 0x16: /* push ss */
2701 emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
2703 case 0x17: /* pop ss */
2704 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
2705 if (rc != X86EMUL_CONTINUE)
2710 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
2712 case 0x1e: /* push ds */
2713 emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
2715 case 0x1f: /* pop ds */
2716 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
2717 if (rc != X86EMUL_CONTINUE)
2722 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
2726 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
2730 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
2734 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2736 case 0x40 ... 0x47: /* inc r16/r32 */
2737 emulate_1op("inc", c->dst, ctxt->eflags);
2739 case 0x48 ... 0x4f: /* dec r16/r32 */
2740 emulate_1op("dec", c->dst, ctxt->eflags);
2742 case 0x50 ... 0x57: /* push reg */
2743 emulate_push(ctxt, ops);
2745 case 0x58 ... 0x5f: /* pop reg */
2747 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
2748 if (rc != X86EMUL_CONTINUE)
2751 case 0x60: /* pusha */
2752 rc = emulate_pusha(ctxt, ops);
2753 if (rc != X86EMUL_CONTINUE)
2756 case 0x61: /* popa */
2757 rc = emulate_popa(ctxt, ops);
2758 if (rc != X86EMUL_CONTINUE)
2761 case 0x63: /* movsxd */
2762 if (ctxt->mode != X86EMUL_MODE_PROT64)
2763 goto cannot_emulate;
2764 c->dst.val = (s32) c->src.val;
2766 case 0x68: /* push imm */
2767 case 0x6a: /* push imm8 */
2768 emulate_push(ctxt, ops);
2770 case 0x6c: /* insb */
2771 case 0x6d: /* insw/insd */
2772 c->dst.bytes = min(c->dst.bytes, 4u);
2773 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2775 emulate_gp(ctxt, 0);
2778 if (!pio_in_emulated(ctxt, ops, c->dst.bytes,
2779 c->regs[VCPU_REGS_RDX], &c->dst.val))
2780 goto done; /* IO is needed, skip writeback */
2782 case 0x6e: /* outsb */
2783 case 0x6f: /* outsw/outsd */
2784 c->src.bytes = min(c->src.bytes, 4u);
2785 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2787 emulate_gp(ctxt, 0);
2790 ops->pio_out_emulated(c->src.bytes, c->regs[VCPU_REGS_RDX],
2791 &c->src.val, 1, ctxt->vcpu);
2793 c->dst.type = OP_NONE; /* nothing to writeback */
2795 case 0x70 ... 0x7f: /* jcc (short) */
2796 if (test_cc(c->b, ctxt->eflags))
2797 jmp_rel(c, c->src.val);
2799 case 0x80 ... 0x83: /* Grp1 */
2800 switch (c->modrm_reg) {
2821 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
2823 case 0x86 ... 0x87: /* xchg */
2825 /* Write back the register source. */
2826 switch (c->dst.bytes) {
2828 *(u8 *) c->src.ptr = (u8) c->dst.val;
2831 *(u16 *) c->src.ptr = (u16) c->dst.val;
2834 *c->src.ptr = (u32) c->dst.val;
2835 break; /* 64b reg: zero-extend */
2837 *c->src.ptr = c->dst.val;
2841 * Write back the memory destination with implicit LOCK
2844 c->dst.val = c->src.val;
2847 case 0x88 ... 0x8b: /* mov */
2849 case 0x8c: /* mov r/m, sreg */
2850 if (c->modrm_reg > VCPU_SREG_GS) {
2854 c->dst.val = ops->get_segment_selector(c->modrm_reg, ctxt->vcpu);
2856 case 0x8d: /* lea r16/r32, m */
2857 c->dst.val = c->modrm_ea;
2859 case 0x8e: { /* mov seg, r/m16 */
2864 if (c->modrm_reg == VCPU_SREG_CS ||
2865 c->modrm_reg > VCPU_SREG_GS) {
2870 if (c->modrm_reg == VCPU_SREG_SS)
2871 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2873 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
2875 c->dst.type = OP_NONE; /* Disable writeback. */
2878 case 0x8f: /* pop (sole member of Grp1a) */
2879 rc = emulate_grp1a(ctxt, ops);
2880 if (rc != X86EMUL_CONTINUE)
2883 case 0x90: /* nop / xchg r8,rax */
2884 if (c->dst.ptr == (unsigned long *)&c->regs[VCPU_REGS_RAX]) {
2885 c->dst.type = OP_NONE; /* nop */
2888 case 0x91 ... 0x97: /* xchg reg,rax */
2889 c->src.type = OP_REG;
2890 c->src.bytes = c->op_bytes;
2891 c->src.ptr = (unsigned long *) &c->regs[VCPU_REGS_RAX];
2892 c->src.val = *(c->src.ptr);
2894 case 0x9c: /* pushf */
2895 c->src.val = (unsigned long) ctxt->eflags;
2896 emulate_push(ctxt, ops);
2898 case 0x9d: /* popf */
2899 c->dst.type = OP_REG;
2900 c->dst.ptr = (unsigned long *) &ctxt->eflags;
2901 c->dst.bytes = c->op_bytes;
2902 rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
2903 if (rc != X86EMUL_CONTINUE)
2906 case 0xa0 ... 0xa3: /* mov */
2907 case 0xa4 ... 0xa5: /* movs */
2909 case 0xa6 ... 0xa7: /* cmps */
2910 c->dst.type = OP_NONE; /* Disable writeback. */
2911 DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.ptr, c->dst.ptr);
2913 case 0xa8 ... 0xa9: /* test ax, imm */
2915 case 0xaa ... 0xab: /* stos */
2916 c->dst.val = c->regs[VCPU_REGS_RAX];
2918 case 0xac ... 0xad: /* lods */
2920 case 0xae ... 0xaf: /* scas */
2921 DPRINTF("Urk! I don't handle SCAS.\n");
2922 goto cannot_emulate;
2923 case 0xb0 ... 0xbf: /* mov r, imm */
2928 case 0xc3: /* ret */
2929 c->dst.type = OP_REG;
2930 c->dst.ptr = &c->eip;
2931 c->dst.bytes = c->op_bytes;
2932 goto pop_instruction;
2933 case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */
2935 c->dst.val = c->src.val;
2937 case 0xcb: /* ret far */
2938 rc = emulate_ret_far(ctxt, ops);
2939 if (rc != X86EMUL_CONTINUE)
2942 case 0xcf: /* iret */
2943 rc = emulate_iret(ctxt, ops);
2945 if (rc != X86EMUL_CONTINUE)
2948 case 0xd0 ... 0xd1: /* Grp2 */
2952 case 0xd2 ... 0xd3: /* Grp2 */
2953 c->src.val = c->regs[VCPU_REGS_RCX];
2956 case 0xe4: /* inb */
2959 case 0xe6: /* outb */
2960 case 0xe7: /* out */
2962 case 0xe8: /* call (near) */ {
2963 long int rel = c->src.val;
2964 c->src.val = (unsigned long) c->eip;
2966 emulate_push(ctxt, ops);
2969 case 0xe9: /* jmp rel */
2971 case 0xea: { /* jmp far */
2974 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
2976 if (load_segment_descriptor(ctxt, ops, sel, VCPU_SREG_CS))
2980 memcpy(&c->eip, c->src.valptr, c->op_bytes);
2984 jmp: /* jmp rel short */
2985 jmp_rel(c, c->src.val);
2986 c->dst.type = OP_NONE; /* Disable writeback. */
2988 case 0xec: /* in al,dx */
2989 case 0xed: /* in (e/r)ax,dx */
2990 c->src.val = c->regs[VCPU_REGS_RDX];
2992 c->dst.bytes = min(c->dst.bytes, 4u);
2993 if (!emulator_io_permited(ctxt, ops, c->src.val, c->dst.bytes)) {
2994 emulate_gp(ctxt, 0);
2997 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
2999 goto done; /* IO is needed */
3001 case 0xee: /* out dx,al */
3002 case 0xef: /* out dx,(e/r)ax */
3003 c->src.val = c->regs[VCPU_REGS_RDX];
3005 c->dst.bytes = min(c->dst.bytes, 4u);
3006 if (!emulator_io_permited(ctxt, ops, c->src.val, c->dst.bytes)) {
3007 emulate_gp(ctxt, 0);
3010 ops->pio_out_emulated(c->dst.bytes, c->src.val, &c->dst.val, 1,
3012 c->dst.type = OP_NONE; /* Disable writeback. */
3014 case 0xf4: /* hlt */
3015 ctxt->vcpu->arch.halt_request = 1;
3017 case 0xf5: /* cmc */
3018 /* complement carry flag from eflags reg */
3019 ctxt->eflags ^= EFLG_CF;
3020 c->dst.type = OP_NONE; /* Disable writeback. */
3022 case 0xf6 ... 0xf7: /* Grp3 */
3023 if (!emulate_grp3(ctxt, ops))
3024 goto cannot_emulate;
3026 case 0xf8: /* clc */
3027 ctxt->eflags &= ~EFLG_CF;
3028 c->dst.type = OP_NONE; /* Disable writeback. */
3030 case 0xfa: /* cli */
3031 if (emulator_bad_iopl(ctxt, ops)) {
3032 emulate_gp(ctxt, 0);
3035 ctxt->eflags &= ~X86_EFLAGS_IF;
3036 c->dst.type = OP_NONE; /* Disable writeback. */
3039 case 0xfb: /* sti */
3040 if (emulator_bad_iopl(ctxt, ops)) {
3041 emulate_gp(ctxt, 0);
3044 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3045 ctxt->eflags |= X86_EFLAGS_IF;
3046 c->dst.type = OP_NONE; /* Disable writeback. */
3049 case 0xfc: /* cld */
3050 ctxt->eflags &= ~EFLG_DF;
3051 c->dst.type = OP_NONE; /* Disable writeback. */
3053 case 0xfd: /* std */
3054 ctxt->eflags |= EFLG_DF;
3055 c->dst.type = OP_NONE; /* Disable writeback. */
3057 case 0xfe: /* Grp4 */
3059 rc = emulate_grp45(ctxt, ops);
3060 if (rc != X86EMUL_CONTINUE)
3063 case 0xff: /* Grp5 */
3064 if (c->modrm_reg == 5)
3068 goto cannot_emulate;
3072 rc = writeback(ctxt, ops);
3073 if (rc != X86EMUL_CONTINUE)
3077 * restore dst type in case the decoding will be reused
3078 * (happens for string instruction )
3080 c->dst.type = saved_dst_type;
3082 if ((c->d & SrcMask) == SrcSI)
3083 string_addr_inc(ctxt, seg_override_base(ctxt, ops, c),
3084 VCPU_REGS_RSI, &c->src);
3086 if ((c->d & DstMask) == DstDI)
3087 string_addr_inc(ctxt, es_base(ctxt, ops), VCPU_REGS_RDI,
3090 if (c->rep_prefix && (c->d & String)) {
3091 struct read_cache *rc = &ctxt->decode.io_read;
3092 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3094 * Re-enter guest when pio read ahead buffer is empty or,
3095 * if it is not used, after each 1024 iteration.
3097 if ((rc->end == 0 && !(c->regs[VCPU_REGS_RCX] & 0x3ff)) ||
3098 (rc->end != 0 && rc->end == rc->pos))
3099 ctxt->restart = false;
3102 * reset read cache here in case string instruction is restared
3105 ctxt->decode.mem_read.end = 0;
3109 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
3113 case 0x01: /* lgdt, lidt, lmsw */
3114 switch (c->modrm_reg) {
3116 unsigned long address;
3118 case 0: /* vmcall */
3119 if (c->modrm_mod != 3 || c->modrm_rm != 1)
3120 goto cannot_emulate;
3122 rc = kvm_fix_hypercall(ctxt->vcpu);
3123 if (rc != X86EMUL_CONTINUE)
3126 /* Let the processor re-execute the fixed hypercall */
3128 /* Disable writeback. */
3129 c->dst.type = OP_NONE;
3132 rc = read_descriptor(ctxt, ops, c->src.ptr,
3133 &size, &address, c->op_bytes);
3134 if (rc != X86EMUL_CONTINUE)
3136 realmode_lgdt(ctxt->vcpu, size, address);
3137 /* Disable writeback. */
3138 c->dst.type = OP_NONE;
3140 case 3: /* lidt/vmmcall */
3141 if (c->modrm_mod == 3) {
3142 switch (c->modrm_rm) {
3144 rc = kvm_fix_hypercall(ctxt->vcpu);
3145 if (rc != X86EMUL_CONTINUE)
3149 goto cannot_emulate;
3152 rc = read_descriptor(ctxt, ops, c->src.ptr,
3155 if (rc != X86EMUL_CONTINUE)
3157 realmode_lidt(ctxt->vcpu, size, address);
3159 /* Disable writeback. */
3160 c->dst.type = OP_NONE;
3164 c->dst.val = ops->get_cr(0, ctxt->vcpu);
3167 ops->set_cr(0, (ops->get_cr(0, ctxt->vcpu) & ~0x0ful) |
3168 (c->src.val & 0x0f), ctxt->vcpu);
3169 c->dst.type = OP_NONE;
3171 case 5: /* not defined */
3175 emulate_invlpg(ctxt->vcpu, c->modrm_ea);
3176 /* Disable writeback. */
3177 c->dst.type = OP_NONE;
3180 goto cannot_emulate;
3183 case 0x05: /* syscall */
3184 rc = emulate_syscall(ctxt, ops);
3185 if (rc != X86EMUL_CONTINUE)
3191 emulate_clts(ctxt->vcpu);
3192 c->dst.type = OP_NONE;
3194 case 0x09: /* wbinvd */
3195 kvm_emulate_wbinvd(ctxt->vcpu);
3196 c->dst.type = OP_NONE;
3198 case 0x08: /* invd */
3199 case 0x0d: /* GrpP (prefetch) */
3200 case 0x18: /* Grp16 (prefetch/nop) */
3201 c->dst.type = OP_NONE;
3203 case 0x20: /* mov cr, reg */
3204 switch (c->modrm_reg) {
3211 c->regs[c->modrm_rm] = ops->get_cr(c->modrm_reg, ctxt->vcpu);
3212 c->dst.type = OP_NONE; /* no writeback */
3214 case 0x21: /* mov from dr to reg */
3215 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
3216 (c->modrm_reg == 4 || c->modrm_reg == 5)) {
3220 ops->get_dr(c->modrm_reg, &c->regs[c->modrm_rm], ctxt->vcpu);
3221 c->dst.type = OP_NONE; /* no writeback */
3223 case 0x22: /* mov reg, cr */
3224 if (ops->set_cr(c->modrm_reg, c->modrm_val, ctxt->vcpu)) {
3225 emulate_gp(ctxt, 0);
3228 c->dst.type = OP_NONE;
3230 case 0x23: /* mov from reg to dr */
3231 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
3232 (c->modrm_reg == 4 || c->modrm_reg == 5)) {
3237 if (ops->set_dr(c->modrm_reg, c->regs[c->modrm_rm] &
3238 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
3239 ~0ULL : ~0U), ctxt->vcpu) < 0) {
3240 /* #UD condition is already handled by the code above */
3241 emulate_gp(ctxt, 0);
3245 c->dst.type = OP_NONE; /* no writeback */
3249 msr_data = (u32)c->regs[VCPU_REGS_RAX]
3250 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
3251 if (ops->set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) {
3252 emulate_gp(ctxt, 0);
3255 rc = X86EMUL_CONTINUE;
3256 c->dst.type = OP_NONE;
3260 if (ops->get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) {
3261 emulate_gp(ctxt, 0);
3264 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
3265 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
3267 rc = X86EMUL_CONTINUE;
3268 c->dst.type = OP_NONE;
3270 case 0x34: /* sysenter */
3271 rc = emulate_sysenter(ctxt, ops);
3272 if (rc != X86EMUL_CONTINUE)
3277 case 0x35: /* sysexit */
3278 rc = emulate_sysexit(ctxt, ops);
3279 if (rc != X86EMUL_CONTINUE)
3284 case 0x40 ... 0x4f: /* cmov */
3285 c->dst.val = c->dst.orig_val = c->src.val;
3286 if (!test_cc(c->b, ctxt->eflags))
3287 c->dst.type = OP_NONE; /* no writeback */
3289 case 0x80 ... 0x8f: /* jnz rel, etc*/
3290 if (test_cc(c->b, ctxt->eflags))
3291 jmp_rel(c, c->src.val);
3292 c->dst.type = OP_NONE;
3294 case 0xa0: /* push fs */
3295 emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
3297 case 0xa1: /* pop fs */
3298 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
3299 if (rc != X86EMUL_CONTINUE)
3304 c->dst.type = OP_NONE;
3305 /* only subword offset */
3306 c->src.val &= (c->dst.bytes << 3) - 1;
3307 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
3309 case 0xa4: /* shld imm8, r, r/m */
3310 case 0xa5: /* shld cl, r, r/m */
3311 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
3313 case 0xa8: /* push gs */
3314 emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
3316 case 0xa9: /* pop gs */
3317 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
3318 if (rc != X86EMUL_CONTINUE)
3323 /* only subword offset */
3324 c->src.val &= (c->dst.bytes << 3) - 1;
3325 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
3327 case 0xac: /* shrd imm8, r, r/m */
3328 case 0xad: /* shrd cl, r, r/m */
3329 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
3331 case 0xae: /* clflush */
3333 case 0xb0 ... 0xb1: /* cmpxchg */
3335 * Save real source value, then compare EAX against
3338 c->src.orig_val = c->src.val;
3339 c->src.val = c->regs[VCPU_REGS_RAX];
3340 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
3341 if (ctxt->eflags & EFLG_ZF) {
3342 /* Success: write back to memory. */
3343 c->dst.val = c->src.orig_val;
3345 /* Failure: write the value we saw to EAX. */
3346 c->dst.type = OP_REG;
3347 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
3352 /* only subword offset */
3353 c->src.val &= (c->dst.bytes << 3) - 1;
3354 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
3356 case 0xb6 ... 0xb7: /* movzx */
3357 c->dst.bytes = c->op_bytes;
3358 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
3361 case 0xba: /* Grp8 */
3362 switch (c->modrm_reg & 3) {
3375 /* only subword offset */
3376 c->src.val &= (c->dst.bytes << 3) - 1;
3377 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
3379 case 0xbe ... 0xbf: /* movsx */
3380 c->dst.bytes = c->op_bytes;
3381 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
3384 case 0xc3: /* movnti */
3385 c->dst.bytes = c->op_bytes;
3386 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
3389 case 0xc7: /* Grp9 (cmpxchg8b) */
3390 rc = emulate_grp9(ctxt, ops);
3391 if (rc != X86EMUL_CONTINUE)
3395 goto cannot_emulate;
3400 DPRINTF("Cannot emulate %02x\n", c->b);