1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affilates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
26 #include <public/xen.h>
27 #define DPRINTF(_f, _a ...) printf(_f , ## _a)
29 #include <linux/kvm_host.h>
30 #include "kvm_cache_regs.h"
31 #define DPRINTF(x...) do {} while (0)
33 #include <linux/module.h>
34 #include <asm/kvm_emulate.h>
40 * Opcode effective-address decode tables.
41 * Note that we only emulate instructions that have at least one memory
42 * operand (excluding implicit stack references). We assume that stack
43 * references and instruction fetches will never occur in special memory
44 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
48 /* Operand sizes: 8-bit operands or specified/overridden size. */
49 #define ByteOp (1<<0) /* 8-bit operands. */
50 /* Destination operand type. */
51 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
52 #define DstReg (2<<1) /* Register operand. */
53 #define DstMem (3<<1) /* Memory operand. */
54 #define DstAcc (4<<1) /* Destination Accumulator */
55 #define DstDI (5<<1) /* Destination is in ES:(E)DI */
56 #define DstMem64 (6<<1) /* 64bit memory operand */
57 #define DstMask (7<<1)
58 /* Source operand type. */
59 #define SrcNone (0<<4) /* No source operand. */
60 #define SrcImplicit (0<<4) /* Source operand is implicit in the opcode. */
61 #define SrcReg (1<<4) /* Register operand. */
62 #define SrcMem (2<<4) /* Memory operand. */
63 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
64 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
65 #define SrcImm (5<<4) /* Immediate operand. */
66 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
67 #define SrcOne (7<<4) /* Implied '1' */
68 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
69 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
70 #define SrcSI (0xa<<4) /* Source is in the DS:RSI */
71 #define SrcImmFAddr (0xb<<4) /* Source is immediate far address */
72 #define SrcMemFAddr (0xc<<4) /* Source is far address in memory */
73 #define SrcAcc (0xd<<4) /* Source Accumulator */
74 #define SrcMask (0xf<<4)
75 /* Generic ModRM decode. */
77 /* Destination is only written; never read. */
80 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
81 #define String (1<<12) /* String instruction (rep capable) */
82 #define Stack (1<<13) /* Stack instruction (push/pop) */
83 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
84 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
86 #define Undefined (1<<25) /* No Such Instruction */
87 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
88 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
90 /* Source 2 operand type */
91 #define Src2None (0<<29)
92 #define Src2CL (1<<29)
93 #define Src2ImmByte (2<<29)
94 #define Src2One (3<<29)
95 #define Src2Mask (7<<29)
98 #define X3(x) X2(x), x
99 #define X4(x) X2(x), X2(x)
100 #define X5(x) X4(x), x
101 #define X6(x) X4(x), X2(x)
102 #define X7(x) X4(x), X3(x)
103 #define X8(x) X4(x), X4(x)
104 #define X16(x) X8(x), X8(x)
109 struct opcode *group;
110 struct group_dual *gdual;
115 struct opcode mod012[8];
116 struct opcode mod3[8];
119 #define D(_y) { .flags = (_y) }
121 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
122 #define GD(_f, _g) { .flags = ((_f) | Group | GroupDual), .u.gdual = (_g) }
124 static struct opcode group1[] = {
128 static struct opcode group1A[] = {
129 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
132 static struct opcode group3[] = {
133 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
134 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
138 static struct opcode group4[] = {
139 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
143 static struct opcode group5[] = {
144 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
145 D(SrcMem | ModRM | Stack), N,
146 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
147 D(SrcMem | ModRM | Stack), N,
150 static struct group_dual group7 = { {
151 N, N, D(ModRM | SrcMem | Priv), D(ModRM | SrcMem | Priv),
152 D(SrcNone | ModRM | DstMem | Mov), N,
153 D(SrcMem16 | ModRM | Mov | Priv), D(SrcMem | ModRM | ByteOp | Priv),
155 D(SrcNone | ModRM | Priv), N, N, D(SrcNone | ModRM | Priv),
156 D(SrcNone | ModRM | DstMem | Mov), N,
157 D(SrcMem16 | ModRM | Mov | Priv), N,
160 static struct opcode group8[] = {
162 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
163 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
166 static struct group_dual group9 = { {
167 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
169 N, N, N, N, N, N, N, N,
172 static struct opcode opcode_table[256] = {
174 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
175 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
176 D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
177 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
179 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
180 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
181 D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
182 D(ImplicitOps | Stack | No64), N,
184 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
185 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
186 D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
187 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
189 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
190 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
191 D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
192 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
194 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
195 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
196 D(ByteOp | DstAcc | SrcImmByte), D(DstAcc | SrcImm), N, N,
198 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
199 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
200 D(ByteOp | DstAcc | SrcImmByte), D(DstAcc | SrcImm), N, N,
202 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
203 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
204 D(ByteOp | DstAcc | SrcImmByte), D(DstAcc | SrcImm), N, N,
206 D(ByteOp | DstMem | SrcReg | ModRM), D(DstMem | SrcReg | ModRM),
207 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
208 D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
213 X8(D(SrcReg | Stack)),
215 X8(D(DstReg | Stack)),
217 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
218 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
221 D(SrcImm | Mov | Stack), N, D(SrcImmByte | Mov | Stack), N,
222 D(DstDI | ByteOp | Mov | String), D(DstDI | Mov | String), /* insb, insw/insd */
223 D(SrcSI | ByteOp | ImplicitOps | String), D(SrcSI | ImplicitOps | String), /* outsb, outsw/outsd */
227 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
228 G(DstMem | SrcImm | ModRM | Group, group1),
229 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
230 G(DstMem | SrcImmByte | ModRM | Group, group1),
231 D(ByteOp | DstMem | SrcReg | ModRM), D(DstMem | SrcReg | ModRM),
232 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
234 D(ByteOp | DstMem | SrcReg | ModRM | Mov), D(DstMem | SrcReg | ModRM | Mov),
235 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem | ModRM | Mov),
236 D(DstMem | SrcNone | ModRM | Mov), D(ModRM | DstReg),
237 D(ImplicitOps | SrcMem16 | ModRM), G(0, group1A),
239 D(DstReg), D(DstReg), D(DstReg), D(DstReg), D(DstReg), D(DstReg), D(DstReg), D(DstReg),
241 N, N, D(SrcImmFAddr | No64), N,
242 D(ImplicitOps | Stack), D(ImplicitOps | Stack), N, N,
244 D(ByteOp | DstAcc | SrcMem | Mov | MemAbs), D(DstAcc | SrcMem | Mov | MemAbs),
245 D(ByteOp | DstMem | SrcAcc | Mov | MemAbs), D(DstMem | SrcAcc | Mov | MemAbs),
246 D(ByteOp | SrcSI | DstDI | Mov | String), D(SrcSI | DstDI | Mov | String),
247 D(ByteOp | SrcSI | DstDI | String), D(SrcSI | DstDI | String),
249 D(DstAcc | SrcImmByte | ByteOp), D(DstAcc | SrcImm), D(ByteOp | DstDI | Mov | String), D(DstDI | Mov | String),
250 D(ByteOp | SrcSI | DstAcc | Mov | String), D(SrcSI | DstAcc | Mov | String),
251 D(ByteOp | DstDI | String), D(DstDI | String),
253 X8(D(ByteOp | DstReg | SrcImm | Mov)),
255 X8(D(DstReg | SrcImm | Mov)),
257 D(ByteOp | DstMem | SrcImm | ModRM), D(DstMem | SrcImmByte | ModRM),
258 N, D(ImplicitOps | Stack), N, N,
259 D(ByteOp | DstMem | SrcImm | ModRM | Mov), D(DstMem | SrcImm | ModRM | Mov),
261 N, N, N, D(ImplicitOps | Stack),
262 D(ImplicitOps), D(SrcImmByte), D(ImplicitOps | No64), D(ImplicitOps),
264 D(ByteOp | DstMem | SrcImplicit | ModRM), D(DstMem | SrcImplicit | ModRM),
265 D(ByteOp | DstMem | SrcImplicit | ModRM), D(DstMem | SrcImplicit | ModRM),
268 N, N, N, N, N, N, N, N,
271 D(ByteOp | SrcImmUByte | DstAcc), D(SrcImmUByte | DstAcc),
272 D(ByteOp | SrcImmUByte | DstAcc), D(SrcImmUByte | DstAcc),
274 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
275 D(SrcImmFAddr | No64), D(SrcImmByte | ImplicitOps),
276 D(SrcNone | ByteOp | DstAcc), D(SrcNone | DstAcc),
277 D(SrcNone | ByteOp | DstAcc), D(SrcNone | DstAcc),
280 D(ImplicitOps | Priv), D(ImplicitOps), G(ByteOp, group3), G(0, group3),
282 D(ImplicitOps), N, D(ImplicitOps), D(ImplicitOps),
283 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
286 static struct opcode twobyte_table[256] = {
288 N, GD(0, &group7), N, N,
289 N, D(ImplicitOps), D(ImplicitOps | Priv), N,
290 D(ImplicitOps | Priv), D(ImplicitOps | Priv), N, N,
291 N, D(ImplicitOps | ModRM), N, N,
293 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
295 D(ModRM | ImplicitOps | Priv), D(ModRM | Priv),
296 D(ModRM | ImplicitOps | Priv), D(ModRM | Priv),
298 N, N, N, N, N, N, N, N,
300 D(ImplicitOps | Priv), N, D(ImplicitOps | Priv), N,
301 D(ImplicitOps), D(ImplicitOps | Priv), N, N,
302 N, N, N, N, N, N, N, N,
304 X16(D(DstReg | SrcMem | ModRM | Mov)),
306 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
308 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
310 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
314 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
316 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
317 N, D(DstMem | SrcReg | ModRM | BitOp),
318 D(DstMem | SrcReg | Src2ImmByte | ModRM),
319 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
321 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
322 N, D(DstMem | SrcReg | ModRM | BitOp | Lock),
323 D(DstMem | SrcReg | Src2ImmByte | ModRM),
324 D(DstMem | SrcReg | Src2CL | ModRM),
327 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
328 N, D(DstMem | SrcReg | ModRM | BitOp | Lock),
329 N, N, D(ByteOp | DstReg | SrcMem | ModRM | Mov),
330 D(DstReg | SrcMem16 | ModRM | Mov),
333 G(0, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
334 N, N, D(ByteOp | DstReg | SrcMem | ModRM | Mov),
335 D(DstReg | SrcMem16 | ModRM | Mov),
337 N, N, N, D(DstMem | SrcReg | ModRM | Mov),
338 N, N, N, GD(0, &group9),
339 N, N, N, N, N, N, N, N,
341 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
343 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
345 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
353 /* EFLAGS bit definitions. */
354 #define EFLG_ID (1<<21)
355 #define EFLG_VIP (1<<20)
356 #define EFLG_VIF (1<<19)
357 #define EFLG_AC (1<<18)
358 #define EFLG_VM (1<<17)
359 #define EFLG_RF (1<<16)
360 #define EFLG_IOPL (3<<12)
361 #define EFLG_NT (1<<14)
362 #define EFLG_OF (1<<11)
363 #define EFLG_DF (1<<10)
364 #define EFLG_IF (1<<9)
365 #define EFLG_TF (1<<8)
366 #define EFLG_SF (1<<7)
367 #define EFLG_ZF (1<<6)
368 #define EFLG_AF (1<<4)
369 #define EFLG_PF (1<<2)
370 #define EFLG_CF (1<<0)
372 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
373 #define EFLG_RESERVED_ONE_MASK 2
376 * Instruction emulation:
377 * Most instructions are emulated directly via a fragment of inline assembly
378 * code. This allows us to save/restore EFLAGS and thus very easily pick up
379 * any modified flags.
382 #if defined(CONFIG_X86_64)
383 #define _LO32 "k" /* force 32-bit operand */
384 #define _STK "%%rsp" /* stack pointer */
385 #elif defined(__i386__)
386 #define _LO32 "" /* force 32-bit operand */
387 #define _STK "%%esp" /* stack pointer */
391 * These EFLAGS bits are restored from saved value during emulation, and
392 * any changes are written back to the saved value after emulation.
394 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
396 /* Before executing instruction: restore necessary bits in EFLAGS. */
397 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
398 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
399 "movl %"_sav",%"_LO32 _tmp"; " \
402 "movl %"_msk",%"_LO32 _tmp"; " \
403 "andl %"_LO32 _tmp",("_STK"); " \
405 "notl %"_LO32 _tmp"; " \
406 "andl %"_LO32 _tmp",("_STK"); " \
407 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
409 "orl %"_LO32 _tmp",("_STK"); " \
413 /* After executing instruction: write-back necessary bits in EFLAGS. */
414 #define _POST_EFLAGS(_sav, _msk, _tmp) \
415 /* _sav |= EFLAGS & _msk; */ \
418 "andl %"_msk",%"_LO32 _tmp"; " \
419 "orl %"_LO32 _tmp",%"_sav"; "
427 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix) \
429 __asm__ __volatile__ ( \
430 _PRE_EFLAGS("0", "4", "2") \
431 _op _suffix " %"_x"3,%1; " \
432 _POST_EFLAGS("0", "4", "2") \
433 : "=m" (_eflags), "=m" ((_dst).val), \
435 : _y ((_src).val), "i" (EFLAGS_MASK)); \
439 /* Raw emulation: instruction has two explicit operands. */
440 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
442 unsigned long _tmp; \
444 switch ((_dst).bytes) { \
446 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w"); \
449 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l"); \
452 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q")); \
457 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
459 unsigned long _tmp; \
460 switch ((_dst).bytes) { \
462 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b"); \
465 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
466 _wx, _wy, _lx, _ly, _qx, _qy); \
471 /* Source operand is byte-sized and may be restricted to just %cl. */
472 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
473 __emulate_2op(_op, _src, _dst, _eflags, \
474 "b", "c", "b", "c", "b", "c", "b", "c")
476 /* Source operand is byte, word, long or quad sized. */
477 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
478 __emulate_2op(_op, _src, _dst, _eflags, \
479 "b", "q", "w", "r", _LO32, "r", "", "r")
481 /* Source operand is word, long or quad sized. */
482 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
483 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
484 "w", "r", _LO32, "r", "", "r")
486 /* Instruction has three operands and one operand is stored in ECX register */
487 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
489 unsigned long _tmp; \
490 _type _clv = (_cl).val; \
491 _type _srcv = (_src).val; \
492 _type _dstv = (_dst).val; \
494 __asm__ __volatile__ ( \
495 _PRE_EFLAGS("0", "5", "2") \
496 _op _suffix " %4,%1 \n" \
497 _POST_EFLAGS("0", "5", "2") \
498 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
499 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
502 (_cl).val = (unsigned long) _clv; \
503 (_src).val = (unsigned long) _srcv; \
504 (_dst).val = (unsigned long) _dstv; \
507 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
509 switch ((_dst).bytes) { \
511 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
512 "w", unsigned short); \
515 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
516 "l", unsigned int); \
519 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
520 "q", unsigned long)); \
525 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
527 unsigned long _tmp; \
529 __asm__ __volatile__ ( \
530 _PRE_EFLAGS("0", "3", "2") \
531 _op _suffix " %1; " \
532 _POST_EFLAGS("0", "3", "2") \
533 : "=m" (_eflags), "+m" ((_dst).val), \
535 : "i" (EFLAGS_MASK)); \
538 /* Instruction has only one explicit operand (no source operand). */
539 #define emulate_1op(_op, _dst, _eflags) \
541 switch ((_dst).bytes) { \
542 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
543 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
544 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
545 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
549 /* Fetch next part of the instruction being emulated. */
550 #define insn_fetch(_type, _size, _eip) \
551 ({ unsigned long _x; \
552 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
553 if (rc != X86EMUL_CONTINUE) \
559 #define insn_fetch_arr(_arr, _size, _eip) \
560 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
561 if (rc != X86EMUL_CONTINUE) \
566 static inline unsigned long ad_mask(struct decode_cache *c)
568 return (1UL << (c->ad_bytes << 3)) - 1;
571 /* Access/update address held in a register, based on addressing mode. */
572 static inline unsigned long
573 address_mask(struct decode_cache *c, unsigned long reg)
575 if (c->ad_bytes == sizeof(unsigned long))
578 return reg & ad_mask(c);
581 static inline unsigned long
582 register_address(struct decode_cache *c, unsigned long base, unsigned long reg)
584 return base + address_mask(c, reg);
588 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
590 if (c->ad_bytes == sizeof(unsigned long))
593 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
596 static inline void jmp_rel(struct decode_cache *c, int rel)
598 register_address_increment(c, &c->eip, rel);
601 static void set_seg_override(struct decode_cache *c, int seg)
603 c->has_seg_override = true;
604 c->seg_override = seg;
607 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
608 struct x86_emulate_ops *ops, int seg)
610 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
613 return ops->get_cached_segment_base(seg, ctxt->vcpu);
616 static unsigned long seg_override_base(struct x86_emulate_ctxt *ctxt,
617 struct x86_emulate_ops *ops,
618 struct decode_cache *c)
620 if (!c->has_seg_override)
623 return seg_base(ctxt, ops, c->seg_override);
626 static unsigned long es_base(struct x86_emulate_ctxt *ctxt,
627 struct x86_emulate_ops *ops)
629 return seg_base(ctxt, ops, VCPU_SREG_ES);
632 static unsigned long ss_base(struct x86_emulate_ctxt *ctxt,
633 struct x86_emulate_ops *ops)
635 return seg_base(ctxt, ops, VCPU_SREG_SS);
638 static void emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
639 u32 error, bool valid)
641 ctxt->exception = vec;
642 ctxt->error_code = error;
643 ctxt->error_code_valid = valid;
644 ctxt->restart = false;
647 static void emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
649 emulate_exception(ctxt, GP_VECTOR, err, true);
652 static void emulate_pf(struct x86_emulate_ctxt *ctxt, unsigned long addr,
656 emulate_exception(ctxt, PF_VECTOR, err, true);
659 static void emulate_ud(struct x86_emulate_ctxt *ctxt)
661 emulate_exception(ctxt, UD_VECTOR, 0, false);
664 static void emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
666 emulate_exception(ctxt, TS_VECTOR, err, true);
669 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
670 struct x86_emulate_ops *ops,
671 unsigned long eip, u8 *dest)
673 struct fetch_cache *fc = &ctxt->decode.fetch;
677 if (eip == fc->end) {
678 cur_size = fc->end - fc->start;
679 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
680 rc = ops->fetch(ctxt->cs_base + eip, fc->data + cur_size,
681 size, ctxt->vcpu, NULL);
682 if (rc != X86EMUL_CONTINUE)
686 *dest = fc->data[eip - fc->start];
687 return X86EMUL_CONTINUE;
690 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
691 struct x86_emulate_ops *ops,
692 unsigned long eip, void *dest, unsigned size)
696 /* x86 instructions are limited to 15 bytes. */
697 if (eip + size - ctxt->eip > 15)
698 return X86EMUL_UNHANDLEABLE;
700 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
701 if (rc != X86EMUL_CONTINUE)
704 return X86EMUL_CONTINUE;
708 * Given the 'reg' portion of a ModRM byte, and a register block, return a
709 * pointer into the block that addresses the relevant register.
710 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
712 static void *decode_register(u8 modrm_reg, unsigned long *regs,
717 p = ®s[modrm_reg];
718 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
719 p = (unsigned char *)®s[modrm_reg & 3] + 1;
723 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
724 struct x86_emulate_ops *ops,
726 u16 *size, unsigned long *address, int op_bytes)
733 rc = ops->read_std((unsigned long)ptr, (unsigned long *)size, 2,
735 if (rc != X86EMUL_CONTINUE)
737 rc = ops->read_std((unsigned long)ptr + 2, address, op_bytes,
742 static int test_cc(unsigned int condition, unsigned int flags)
746 switch ((condition & 15) >> 1) {
748 rc |= (flags & EFLG_OF);
750 case 1: /* b/c/nae */
751 rc |= (flags & EFLG_CF);
754 rc |= (flags & EFLG_ZF);
757 rc |= (flags & (EFLG_CF|EFLG_ZF));
760 rc |= (flags & EFLG_SF);
763 rc |= (flags & EFLG_PF);
766 rc |= (flags & EFLG_ZF);
769 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
773 /* Odd condition identifiers (lsb == 1) have inverted sense. */
774 return (!!rc ^ (condition & 1));
777 static void decode_register_operand(struct operand *op,
778 struct decode_cache *c,
781 unsigned reg = c->modrm_reg;
782 int highbyte_regs = c->rex_prefix == 0;
785 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
787 if ((c->d & ByteOp) && !inhibit_bytereg) {
788 op->ptr = decode_register(reg, c->regs, highbyte_regs);
789 op->val = *(u8 *)op->ptr;
792 op->ptr = decode_register(reg, c->regs, 0);
793 op->bytes = c->op_bytes;
796 op->val = *(u16 *)op->ptr;
799 op->val = *(u32 *)op->ptr;
802 op->val = *(u64 *) op->ptr;
806 op->orig_val = op->val;
809 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
810 struct x86_emulate_ops *ops)
812 struct decode_cache *c = &ctxt->decode;
814 int index_reg = 0, base_reg = 0, scale;
815 int rc = X86EMUL_CONTINUE;
818 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
819 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
820 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
823 c->modrm = insn_fetch(u8, 1, c->eip);
824 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
825 c->modrm_reg |= (c->modrm & 0x38) >> 3;
826 c->modrm_rm |= (c->modrm & 0x07);
830 if (c->modrm_mod == 3) {
831 c->modrm_ptr = decode_register(c->modrm_rm,
832 c->regs, c->d & ByteOp);
833 c->modrm_val = *(unsigned long *)c->modrm_ptr;
837 if (c->ad_bytes == 2) {
838 unsigned bx = c->regs[VCPU_REGS_RBX];
839 unsigned bp = c->regs[VCPU_REGS_RBP];
840 unsigned si = c->regs[VCPU_REGS_RSI];
841 unsigned di = c->regs[VCPU_REGS_RDI];
843 /* 16-bit ModR/M decode. */
844 switch (c->modrm_mod) {
846 if (c->modrm_rm == 6)
847 c->modrm_ea += insn_fetch(u16, 2, c->eip);
850 c->modrm_ea += insn_fetch(s8, 1, c->eip);
853 c->modrm_ea += insn_fetch(u16, 2, c->eip);
856 switch (c->modrm_rm) {
858 c->modrm_ea += bx + si;
861 c->modrm_ea += bx + di;
864 c->modrm_ea += bp + si;
867 c->modrm_ea += bp + di;
876 if (c->modrm_mod != 0)
883 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
884 (c->modrm_rm == 6 && c->modrm_mod != 0))
885 if (!c->has_seg_override)
886 set_seg_override(c, VCPU_SREG_SS);
887 c->modrm_ea = (u16)c->modrm_ea;
889 /* 32/64-bit ModR/M decode. */
890 if ((c->modrm_rm & 7) == 4) {
891 sib = insn_fetch(u8, 1, c->eip);
892 index_reg |= (sib >> 3) & 7;
896 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
897 c->modrm_ea += insn_fetch(s32, 4, c->eip);
899 c->modrm_ea += c->regs[base_reg];
901 c->modrm_ea += c->regs[index_reg] << scale;
902 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
903 if (ctxt->mode == X86EMUL_MODE_PROT64)
906 c->modrm_ea += c->regs[c->modrm_rm];
907 switch (c->modrm_mod) {
909 if (c->modrm_rm == 5)
910 c->modrm_ea += insn_fetch(s32, 4, c->eip);
913 c->modrm_ea += insn_fetch(s8, 1, c->eip);
916 c->modrm_ea += insn_fetch(s32, 4, c->eip);
924 static int decode_abs(struct x86_emulate_ctxt *ctxt,
925 struct x86_emulate_ops *ops)
927 struct decode_cache *c = &ctxt->decode;
928 int rc = X86EMUL_CONTINUE;
930 switch (c->ad_bytes) {
932 c->modrm_ea = insn_fetch(u16, 2, c->eip);
935 c->modrm_ea = insn_fetch(u32, 4, c->eip);
938 c->modrm_ea = insn_fetch(u64, 8, c->eip);
946 x86_decode_insn(struct x86_emulate_ctxt *ctxt)
948 struct x86_emulate_ops *ops = ctxt->ops;
949 struct decode_cache *c = &ctxt->decode;
950 int rc = X86EMUL_CONTINUE;
951 int mode = ctxt->mode;
952 int def_op_bytes, def_ad_bytes, dual, goffset;
953 struct opcode opcode, *g_mod012, *g_mod3;
955 /* we cannot decode insn before we complete previous rep insn */
956 WARN_ON(ctxt->restart);
959 c->fetch.start = c->fetch.end = c->eip;
960 ctxt->cs_base = seg_base(ctxt, ops, VCPU_SREG_CS);
963 case X86EMUL_MODE_REAL:
964 case X86EMUL_MODE_VM86:
965 case X86EMUL_MODE_PROT16:
966 def_op_bytes = def_ad_bytes = 2;
968 case X86EMUL_MODE_PROT32:
969 def_op_bytes = def_ad_bytes = 4;
972 case X86EMUL_MODE_PROT64:
981 c->op_bytes = def_op_bytes;
982 c->ad_bytes = def_ad_bytes;
984 /* Legacy prefixes. */
986 switch (c->b = insn_fetch(u8, 1, c->eip)) {
987 case 0x66: /* operand-size override */
988 /* switch between 2/4 bytes */
989 c->op_bytes = def_op_bytes ^ 6;
991 case 0x67: /* address-size override */
992 if (mode == X86EMUL_MODE_PROT64)
993 /* switch between 4/8 bytes */
994 c->ad_bytes = def_ad_bytes ^ 12;
996 /* switch between 2/4 bytes */
997 c->ad_bytes = def_ad_bytes ^ 6;
999 case 0x26: /* ES override */
1000 case 0x2e: /* CS override */
1001 case 0x36: /* SS override */
1002 case 0x3e: /* DS override */
1003 set_seg_override(c, (c->b >> 3) & 3);
1005 case 0x64: /* FS override */
1006 case 0x65: /* GS override */
1007 set_seg_override(c, c->b & 7);
1009 case 0x40 ... 0x4f: /* REX */
1010 if (mode != X86EMUL_MODE_PROT64)
1012 c->rex_prefix = c->b;
1014 case 0xf0: /* LOCK */
1017 case 0xf2: /* REPNE/REPNZ */
1018 c->rep_prefix = REPNE_PREFIX;
1020 case 0xf3: /* REP/REPE/REPZ */
1021 c->rep_prefix = REPE_PREFIX;
1027 /* Any legacy prefix after a REX prefix nullifies its effect. */
1036 if (c->rex_prefix & 8)
1037 c->op_bytes = 8; /* REX.W */
1039 /* Opcode byte(s). */
1040 opcode = opcode_table[c->b];
1041 if (opcode.flags == 0) {
1042 /* Two-byte opcode? */
1045 c->b = insn_fetch(u8, 1, c->eip);
1046 opcode = twobyte_table[c->b];
1049 c->d = opcode.flags;
1052 dual = c->d & GroupDual;
1053 c->modrm = insn_fetch(u8, 1, c->eip);
1056 if (c->d & GroupDual) {
1057 g_mod012 = opcode.u.gdual->mod012;
1058 g_mod3 = opcode.u.gdual->mod3;
1060 g_mod012 = g_mod3 = opcode.u.group;
1062 c->d &= ~(Group | GroupDual);
1064 goffset = (c->modrm >> 3) & 7;
1066 if ((c->modrm >> 6) == 3)
1067 opcode = g_mod3[goffset];
1069 opcode = g_mod012[goffset];
1070 c->d |= opcode.flags;
1074 if (c->d == 0 || (c->d & Undefined)) {
1075 DPRINTF("Cannot emulate %02x\n", c->b);
1079 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
1082 /* ModRM and SIB bytes. */
1084 rc = decode_modrm(ctxt, ops);
1085 else if (c->d & MemAbs)
1086 rc = decode_abs(ctxt, ops);
1087 if (rc != X86EMUL_CONTINUE)
1090 if (!c->has_seg_override)
1091 set_seg_override(c, VCPU_SREG_DS);
1093 if (!(!c->twobyte && c->b == 0x8d))
1094 c->modrm_ea += seg_override_base(ctxt, ops, c);
1096 if (c->ad_bytes != 8)
1097 c->modrm_ea = (u32)c->modrm_ea;
1099 if (c->rip_relative)
1100 c->modrm_ea += c->eip;
1103 * Decode and fetch the source operand: register, memory
1106 switch (c->d & SrcMask) {
1110 decode_register_operand(&c->src, c, 0);
1119 c->src.bytes = (c->d & ByteOp) ? 1 :
1121 /* Don't fetch the address for invlpg: it could be unmapped. */
1122 if (c->twobyte && c->b == 0x01 && c->modrm_reg == 7)
1126 * For instructions with a ModR/M byte, switch to register
1127 * access if Mod = 3.
1129 if ((c->d & ModRM) && c->modrm_mod == 3) {
1130 c->src.type = OP_REG;
1131 c->src.val = c->modrm_val;
1132 c->src.ptr = c->modrm_ptr;
1135 c->src.type = OP_MEM;
1136 c->src.ptr = (unsigned long *)c->modrm_ea;
1141 c->src.type = OP_IMM;
1142 c->src.ptr = (unsigned long *)c->eip;
1143 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1144 if (c->src.bytes == 8)
1146 /* NB. Immediates are sign-extended as necessary. */
1147 switch (c->src.bytes) {
1149 c->src.val = insn_fetch(s8, 1, c->eip);
1152 c->src.val = insn_fetch(s16, 2, c->eip);
1155 c->src.val = insn_fetch(s32, 4, c->eip);
1158 if ((c->d & SrcMask) == SrcImmU) {
1159 switch (c->src.bytes) {
1164 c->src.val &= 0xffff;
1167 c->src.val &= 0xffffffff;
1174 c->src.type = OP_IMM;
1175 c->src.ptr = (unsigned long *)c->eip;
1177 if ((c->d & SrcMask) == SrcImmByte)
1178 c->src.val = insn_fetch(s8, 1, c->eip);
1180 c->src.val = insn_fetch(u8, 1, c->eip);
1183 c->src.type = OP_REG;
1184 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1185 c->src.ptr = &c->regs[VCPU_REGS_RAX];
1186 switch (c->src.bytes) {
1188 c->src.val = *(u8 *)c->src.ptr;
1191 c->src.val = *(u16 *)c->src.ptr;
1194 c->src.val = *(u32 *)c->src.ptr;
1197 c->src.val = *(u64 *)c->src.ptr;
1206 c->src.type = OP_MEM;
1207 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1208 c->src.ptr = (unsigned long *)
1209 register_address(c, seg_override_base(ctxt, ops, c),
1210 c->regs[VCPU_REGS_RSI]);
1214 c->src.type = OP_IMM;
1215 c->src.ptr = (unsigned long *)c->eip;
1216 c->src.bytes = c->op_bytes + 2;
1217 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
1220 c->src.type = OP_MEM;
1221 c->src.ptr = (unsigned long *)c->modrm_ea;
1222 c->src.bytes = c->op_bytes + 2;
1227 * Decode and fetch the second source operand: register, memory
1230 switch (c->d & Src2Mask) {
1235 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
1238 c->src2.type = OP_IMM;
1239 c->src2.ptr = (unsigned long *)c->eip;
1241 c->src2.val = insn_fetch(u8, 1, c->eip);
1249 /* Decode and fetch the destination operand: register or memory. */
1250 switch (c->d & DstMask) {
1252 /* Special instructions do their own operand decoding. */
1255 decode_register_operand(&c->dst, c,
1256 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
1260 if ((c->d & ModRM) && c->modrm_mod == 3) {
1261 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1262 c->dst.type = OP_REG;
1263 c->dst.val = c->dst.orig_val = c->modrm_val;
1264 c->dst.ptr = c->modrm_ptr;
1267 c->dst.type = OP_MEM;
1268 c->dst.ptr = (unsigned long *)c->modrm_ea;
1269 if ((c->d & DstMask) == DstMem64)
1272 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1275 unsigned long mask = ~(c->dst.bytes * 8 - 1);
1277 c->dst.ptr = (void *)c->dst.ptr +
1278 (c->src.val & mask) / 8;
1282 c->dst.type = OP_REG;
1283 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1284 c->dst.ptr = &c->regs[VCPU_REGS_RAX];
1285 switch (c->dst.bytes) {
1287 c->dst.val = *(u8 *)c->dst.ptr;
1290 c->dst.val = *(u16 *)c->dst.ptr;
1293 c->dst.val = *(u32 *)c->dst.ptr;
1296 c->dst.val = *(u64 *)c->dst.ptr;
1299 c->dst.orig_val = c->dst.val;
1302 c->dst.type = OP_MEM;
1303 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1304 c->dst.ptr = (unsigned long *)
1305 register_address(c, es_base(ctxt, ops),
1306 c->regs[VCPU_REGS_RDI]);
1312 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
1315 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1316 struct x86_emulate_ops *ops,
1317 unsigned long addr, void *dest, unsigned size)
1320 struct read_cache *mc = &ctxt->decode.mem_read;
1324 int n = min(size, 8u);
1326 if (mc->pos < mc->end)
1329 rc = ops->read_emulated(addr, mc->data + mc->end, n, &err,
1331 if (rc == X86EMUL_PROPAGATE_FAULT)
1332 emulate_pf(ctxt, addr, err);
1333 if (rc != X86EMUL_CONTINUE)
1338 memcpy(dest, mc->data + mc->pos, n);
1343 return X86EMUL_CONTINUE;
1346 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1347 struct x86_emulate_ops *ops,
1348 unsigned int size, unsigned short port,
1351 struct read_cache *rc = &ctxt->decode.io_read;
1353 if (rc->pos == rc->end) { /* refill pio read ahead */
1354 struct decode_cache *c = &ctxt->decode;
1355 unsigned int in_page, n;
1356 unsigned int count = c->rep_prefix ?
1357 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
1358 in_page = (ctxt->eflags & EFLG_DF) ?
1359 offset_in_page(c->regs[VCPU_REGS_RDI]) :
1360 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
1361 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1365 rc->pos = rc->end = 0;
1366 if (!ops->pio_in_emulated(size, port, rc->data, n, ctxt->vcpu))
1371 memcpy(dest, rc->data + rc->pos, size);
1376 static u32 desc_limit_scaled(struct desc_struct *desc)
1378 u32 limit = get_desc_limit(desc);
1380 return desc->g ? (limit << 12) | 0xfff : limit;
1383 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1384 struct x86_emulate_ops *ops,
1385 u16 selector, struct desc_ptr *dt)
1387 if (selector & 1 << 2) {
1388 struct desc_struct desc;
1389 memset (dt, 0, sizeof *dt);
1390 if (!ops->get_cached_descriptor(&desc, VCPU_SREG_LDTR, ctxt->vcpu))
1393 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1394 dt->address = get_desc_base(&desc);
1396 ops->get_gdt(dt, ctxt->vcpu);
1399 /* allowed just for 8 bytes segments */
1400 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1401 struct x86_emulate_ops *ops,
1402 u16 selector, struct desc_struct *desc)
1405 u16 index = selector >> 3;
1410 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1412 if (dt.size < index * 8 + 7) {
1413 emulate_gp(ctxt, selector & 0xfffc);
1414 return X86EMUL_PROPAGATE_FAULT;
1416 addr = dt.address + index * 8;
1417 ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu, &err);
1418 if (ret == X86EMUL_PROPAGATE_FAULT)
1419 emulate_pf(ctxt, addr, err);
1424 /* allowed just for 8 bytes segments */
1425 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1426 struct x86_emulate_ops *ops,
1427 u16 selector, struct desc_struct *desc)
1430 u16 index = selector >> 3;
1435 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1437 if (dt.size < index * 8 + 7) {
1438 emulate_gp(ctxt, selector & 0xfffc);
1439 return X86EMUL_PROPAGATE_FAULT;
1442 addr = dt.address + index * 8;
1443 ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu, &err);
1444 if (ret == X86EMUL_PROPAGATE_FAULT)
1445 emulate_pf(ctxt, addr, err);
1450 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1451 struct x86_emulate_ops *ops,
1452 u16 selector, int seg)
1454 struct desc_struct seg_desc;
1456 unsigned err_vec = GP_VECTOR;
1458 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1461 memset(&seg_desc, 0, sizeof seg_desc);
1463 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1464 || ctxt->mode == X86EMUL_MODE_REAL) {
1465 /* set real mode segment descriptor */
1466 set_desc_base(&seg_desc, selector << 4);
1467 set_desc_limit(&seg_desc, 0xffff);
1474 /* NULL selector is not valid for TR, CS and SS */
1475 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1479 /* TR should be in GDT only */
1480 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1483 if (null_selector) /* for NULL selector skip all following checks */
1486 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1487 if (ret != X86EMUL_CONTINUE)
1490 err_code = selector & 0xfffc;
1491 err_vec = GP_VECTOR;
1493 /* can't load system descriptor into segment selecor */
1494 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1498 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1504 cpl = ops->cpl(ctxt->vcpu);
1509 * segment is not a writable data segment or segment
1510 * selector's RPL != CPL or segment selector's RPL != CPL
1512 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1516 if (!(seg_desc.type & 8))
1519 if (seg_desc.type & 4) {
1525 if (rpl > cpl || dpl != cpl)
1528 /* CS(RPL) <- CPL */
1529 selector = (selector & 0xfffc) | cpl;
1532 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1535 case VCPU_SREG_LDTR:
1536 if (seg_desc.s || seg_desc.type != 2)
1539 default: /* DS, ES, FS, or GS */
1541 * segment is not a data or readable code segment or
1542 * ((segment is a data or nonconforming code segment)
1543 * and (both RPL and CPL > DPL))
1545 if ((seg_desc.type & 0xa) == 0x8 ||
1546 (((seg_desc.type & 0xc) != 0xc) &&
1547 (rpl > dpl && cpl > dpl)))
1553 /* mark segment as accessed */
1555 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1556 if (ret != X86EMUL_CONTINUE)
1560 ops->set_segment_selector(selector, seg, ctxt->vcpu);
1561 ops->set_cached_descriptor(&seg_desc, seg, ctxt->vcpu);
1562 return X86EMUL_CONTINUE;
1564 emulate_exception(ctxt, err_vec, err_code, true);
1565 return X86EMUL_PROPAGATE_FAULT;
1568 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1569 struct x86_emulate_ops *ops)
1572 struct decode_cache *c = &ctxt->decode;
1575 switch (c->dst.type) {
1577 /* The 4-byte case *is* correct:
1578 * in 64-bit mode we zero-extend.
1580 switch (c->dst.bytes) {
1582 *(u8 *)c->dst.ptr = (u8)c->dst.val;
1585 *(u16 *)c->dst.ptr = (u16)c->dst.val;
1588 *c->dst.ptr = (u32)c->dst.val;
1589 break; /* 64b: zero-ext */
1591 *c->dst.ptr = c->dst.val;
1597 rc = ops->cmpxchg_emulated(
1598 (unsigned long)c->dst.ptr,
1605 rc = ops->write_emulated(
1606 (unsigned long)c->dst.ptr,
1611 if (rc == X86EMUL_PROPAGATE_FAULT)
1613 (unsigned long)c->dst.ptr, err);
1614 if (rc != X86EMUL_CONTINUE)
1623 return X86EMUL_CONTINUE;
1626 static inline void emulate_push(struct x86_emulate_ctxt *ctxt,
1627 struct x86_emulate_ops *ops)
1629 struct decode_cache *c = &ctxt->decode;
1631 c->dst.type = OP_MEM;
1632 c->dst.bytes = c->op_bytes;
1633 c->dst.val = c->src.val;
1634 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1635 c->dst.ptr = (void *) register_address(c, ss_base(ctxt, ops),
1636 c->regs[VCPU_REGS_RSP]);
1639 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1640 struct x86_emulate_ops *ops,
1641 void *dest, int len)
1643 struct decode_cache *c = &ctxt->decode;
1646 rc = read_emulated(ctxt, ops, register_address(c, ss_base(ctxt, ops),
1647 c->regs[VCPU_REGS_RSP]),
1649 if (rc != X86EMUL_CONTINUE)
1652 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1656 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1657 struct x86_emulate_ops *ops,
1658 void *dest, int len)
1661 unsigned long val, change_mask;
1662 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1663 int cpl = ops->cpl(ctxt->vcpu);
1665 rc = emulate_pop(ctxt, ops, &val, len);
1666 if (rc != X86EMUL_CONTINUE)
1669 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1670 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1672 switch(ctxt->mode) {
1673 case X86EMUL_MODE_PROT64:
1674 case X86EMUL_MODE_PROT32:
1675 case X86EMUL_MODE_PROT16:
1677 change_mask |= EFLG_IOPL;
1679 change_mask |= EFLG_IF;
1681 case X86EMUL_MODE_VM86:
1683 emulate_gp(ctxt, 0);
1684 return X86EMUL_PROPAGATE_FAULT;
1686 change_mask |= EFLG_IF;
1688 default: /* real mode */
1689 change_mask |= (EFLG_IOPL | EFLG_IF);
1693 *(unsigned long *)dest =
1694 (ctxt->eflags & ~change_mask) | (val & change_mask);
1699 static void emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1700 struct x86_emulate_ops *ops, int seg)
1702 struct decode_cache *c = &ctxt->decode;
1704 c->src.val = ops->get_segment_selector(seg, ctxt->vcpu);
1706 emulate_push(ctxt, ops);
1709 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1710 struct x86_emulate_ops *ops, int seg)
1712 struct decode_cache *c = &ctxt->decode;
1713 unsigned long selector;
1716 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1717 if (rc != X86EMUL_CONTINUE)
1720 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1724 static int emulate_pusha(struct x86_emulate_ctxt *ctxt,
1725 struct x86_emulate_ops *ops)
1727 struct decode_cache *c = &ctxt->decode;
1728 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1729 int rc = X86EMUL_CONTINUE;
1730 int reg = VCPU_REGS_RAX;
1732 while (reg <= VCPU_REGS_RDI) {
1733 (reg == VCPU_REGS_RSP) ?
1734 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1736 emulate_push(ctxt, ops);
1738 rc = writeback(ctxt, ops);
1739 if (rc != X86EMUL_CONTINUE)
1745 /* Disable writeback. */
1746 c->dst.type = OP_NONE;
1751 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1752 struct x86_emulate_ops *ops)
1754 struct decode_cache *c = &ctxt->decode;
1755 int rc = X86EMUL_CONTINUE;
1756 int reg = VCPU_REGS_RDI;
1758 while (reg >= VCPU_REGS_RAX) {
1759 if (reg == VCPU_REGS_RSP) {
1760 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1765 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1766 if (rc != X86EMUL_CONTINUE)
1773 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
1774 struct x86_emulate_ops *ops)
1776 struct decode_cache *c = &ctxt->decode;
1777 int rc = X86EMUL_CONTINUE;
1778 unsigned long temp_eip = 0;
1779 unsigned long temp_eflags = 0;
1780 unsigned long cs = 0;
1781 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1782 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1783 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1784 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1786 /* TODO: Add stack limit check */
1788 rc = emulate_pop(ctxt, ops, &temp_eip, c->op_bytes);
1790 if (rc != X86EMUL_CONTINUE)
1793 if (temp_eip & ~0xffff) {
1794 emulate_gp(ctxt, 0);
1795 return X86EMUL_PROPAGATE_FAULT;
1798 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1800 if (rc != X86EMUL_CONTINUE)
1803 rc = emulate_pop(ctxt, ops, &temp_eflags, c->op_bytes);
1805 if (rc != X86EMUL_CONTINUE)
1808 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1810 if (rc != X86EMUL_CONTINUE)
1816 if (c->op_bytes == 4)
1817 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1818 else if (c->op_bytes == 2) {
1819 ctxt->eflags &= ~0xffff;
1820 ctxt->eflags |= temp_eflags;
1823 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1824 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1829 static inline int emulate_iret(struct x86_emulate_ctxt *ctxt,
1830 struct x86_emulate_ops* ops)
1832 switch(ctxt->mode) {
1833 case X86EMUL_MODE_REAL:
1834 return emulate_iret_real(ctxt, ops);
1835 case X86EMUL_MODE_VM86:
1836 case X86EMUL_MODE_PROT16:
1837 case X86EMUL_MODE_PROT32:
1838 case X86EMUL_MODE_PROT64:
1840 /* iret from protected mode unimplemented yet */
1841 return X86EMUL_UNHANDLEABLE;
1845 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1846 struct x86_emulate_ops *ops)
1848 struct decode_cache *c = &ctxt->decode;
1850 return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1853 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1855 struct decode_cache *c = &ctxt->decode;
1856 switch (c->modrm_reg) {
1858 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1861 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1864 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1867 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1869 case 4: /* sal/shl */
1870 case 6: /* sal/shl */
1871 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1874 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1877 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1882 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1883 struct x86_emulate_ops *ops)
1885 struct decode_cache *c = &ctxt->decode;
1887 switch (c->modrm_reg) {
1888 case 0 ... 1: /* test */
1889 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1892 c->dst.val = ~c->dst.val;
1895 emulate_1op("neg", c->dst, ctxt->eflags);
1903 static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
1904 struct x86_emulate_ops *ops)
1906 struct decode_cache *c = &ctxt->decode;
1908 switch (c->modrm_reg) {
1910 emulate_1op("inc", c->dst, ctxt->eflags);
1913 emulate_1op("dec", c->dst, ctxt->eflags);
1915 case 2: /* call near abs */ {
1918 c->eip = c->src.val;
1919 c->src.val = old_eip;
1920 emulate_push(ctxt, ops);
1923 case 4: /* jmp abs */
1924 c->eip = c->src.val;
1927 emulate_push(ctxt, ops);
1930 return X86EMUL_CONTINUE;
1933 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1934 struct x86_emulate_ops *ops)
1936 struct decode_cache *c = &ctxt->decode;
1937 u64 old = c->dst.orig_val64;
1939 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1940 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1941 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1942 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1943 ctxt->eflags &= ~EFLG_ZF;
1945 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1946 (u32) c->regs[VCPU_REGS_RBX];
1948 ctxt->eflags |= EFLG_ZF;
1950 return X86EMUL_CONTINUE;
1953 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1954 struct x86_emulate_ops *ops)
1956 struct decode_cache *c = &ctxt->decode;
1960 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1961 if (rc != X86EMUL_CONTINUE)
1963 if (c->op_bytes == 4)
1964 c->eip = (u32)c->eip;
1965 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1966 if (rc != X86EMUL_CONTINUE)
1968 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1973 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1974 struct x86_emulate_ops *ops, struct desc_struct *cs,
1975 struct desc_struct *ss)
1977 memset(cs, 0, sizeof(struct desc_struct));
1978 ops->get_cached_descriptor(cs, VCPU_SREG_CS, ctxt->vcpu);
1979 memset(ss, 0, sizeof(struct desc_struct));
1981 cs->l = 0; /* will be adjusted later */
1982 set_desc_base(cs, 0); /* flat segment */
1983 cs->g = 1; /* 4kb granularity */
1984 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1985 cs->type = 0x0b; /* Read, Execute, Accessed */
1987 cs->dpl = 0; /* will be adjusted later */
1991 set_desc_base(ss, 0); /* flat segment */
1992 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1993 ss->g = 1; /* 4kb granularity */
1995 ss->type = 0x03; /* Read/Write, Accessed */
1996 ss->d = 1; /* 32bit stack segment */
2002 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2004 struct decode_cache *c = &ctxt->decode;
2005 struct desc_struct cs, ss;
2009 /* syscall is not available in real mode */
2010 if (ctxt->mode == X86EMUL_MODE_REAL ||
2011 ctxt->mode == X86EMUL_MODE_VM86) {
2013 return X86EMUL_PROPAGATE_FAULT;
2016 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2018 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
2020 cs_sel = (u16)(msr_data & 0xfffc);
2021 ss_sel = (u16)(msr_data + 8);
2023 if (is_long_mode(ctxt->vcpu)) {
2027 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
2028 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
2029 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
2030 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
2032 c->regs[VCPU_REGS_RCX] = c->eip;
2033 if (is_long_mode(ctxt->vcpu)) {
2034 #ifdef CONFIG_X86_64
2035 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
2037 ops->get_msr(ctxt->vcpu,
2038 ctxt->mode == X86EMUL_MODE_PROT64 ?
2039 MSR_LSTAR : MSR_CSTAR, &msr_data);
2042 ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
2043 ctxt->eflags &= ~(msr_data | EFLG_RF);
2047 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
2048 c->eip = (u32)msr_data;
2050 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2053 return X86EMUL_CONTINUE;
2057 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2059 struct decode_cache *c = &ctxt->decode;
2060 struct desc_struct cs, ss;
2064 /* inject #GP if in real mode */
2065 if (ctxt->mode == X86EMUL_MODE_REAL) {
2066 emulate_gp(ctxt, 0);
2067 return X86EMUL_PROPAGATE_FAULT;
2070 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2071 * Therefore, we inject an #UD.
2073 if (ctxt->mode == X86EMUL_MODE_PROT64) {
2075 return X86EMUL_PROPAGATE_FAULT;
2078 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2080 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
2081 switch (ctxt->mode) {
2082 case X86EMUL_MODE_PROT32:
2083 if ((msr_data & 0xfffc) == 0x0) {
2084 emulate_gp(ctxt, 0);
2085 return X86EMUL_PROPAGATE_FAULT;
2088 case X86EMUL_MODE_PROT64:
2089 if (msr_data == 0x0) {
2090 emulate_gp(ctxt, 0);
2091 return X86EMUL_PROPAGATE_FAULT;
2096 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2097 cs_sel = (u16)msr_data;
2098 cs_sel &= ~SELECTOR_RPL_MASK;
2099 ss_sel = cs_sel + 8;
2100 ss_sel &= ~SELECTOR_RPL_MASK;
2101 if (ctxt->mode == X86EMUL_MODE_PROT64
2102 || is_long_mode(ctxt->vcpu)) {
2107 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
2108 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
2109 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
2110 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
2112 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
2115 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
2116 c->regs[VCPU_REGS_RSP] = msr_data;
2118 return X86EMUL_CONTINUE;
2122 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2124 struct decode_cache *c = &ctxt->decode;
2125 struct desc_struct cs, ss;
2130 /* inject #GP if in real mode or Virtual 8086 mode */
2131 if (ctxt->mode == X86EMUL_MODE_REAL ||
2132 ctxt->mode == X86EMUL_MODE_VM86) {
2133 emulate_gp(ctxt, 0);
2134 return X86EMUL_PROPAGATE_FAULT;
2137 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2139 if ((c->rex_prefix & 0x8) != 0x0)
2140 usermode = X86EMUL_MODE_PROT64;
2142 usermode = X86EMUL_MODE_PROT32;
2146 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
2148 case X86EMUL_MODE_PROT32:
2149 cs_sel = (u16)(msr_data + 16);
2150 if ((msr_data & 0xfffc) == 0x0) {
2151 emulate_gp(ctxt, 0);
2152 return X86EMUL_PROPAGATE_FAULT;
2154 ss_sel = (u16)(msr_data + 24);
2156 case X86EMUL_MODE_PROT64:
2157 cs_sel = (u16)(msr_data + 32);
2158 if (msr_data == 0x0) {
2159 emulate_gp(ctxt, 0);
2160 return X86EMUL_PROPAGATE_FAULT;
2162 ss_sel = cs_sel + 8;
2167 cs_sel |= SELECTOR_RPL_MASK;
2168 ss_sel |= SELECTOR_RPL_MASK;
2170 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
2171 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
2172 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
2173 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
2175 c->eip = c->regs[VCPU_REGS_RDX];
2176 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
2178 return X86EMUL_CONTINUE;
2181 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
2182 struct x86_emulate_ops *ops)
2185 if (ctxt->mode == X86EMUL_MODE_REAL)
2187 if (ctxt->mode == X86EMUL_MODE_VM86)
2189 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2190 return ops->cpl(ctxt->vcpu) > iopl;
2193 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2194 struct x86_emulate_ops *ops,
2197 struct desc_struct tr_seg;
2200 u8 perm, bit_idx = port & 0x7;
2201 unsigned mask = (1 << len) - 1;
2203 ops->get_cached_descriptor(&tr_seg, VCPU_SREG_TR, ctxt->vcpu);
2206 if (desc_limit_scaled(&tr_seg) < 103)
2208 r = ops->read_std(get_desc_base(&tr_seg) + 102, &io_bitmap_ptr, 2,
2210 if (r != X86EMUL_CONTINUE)
2212 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2214 r = ops->read_std(get_desc_base(&tr_seg) + io_bitmap_ptr + port/8,
2215 &perm, 1, ctxt->vcpu, NULL);
2216 if (r != X86EMUL_CONTINUE)
2218 if ((perm >> bit_idx) & mask)
2223 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2224 struct x86_emulate_ops *ops,
2227 if (emulator_bad_iopl(ctxt, ops))
2228 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
2233 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2234 struct x86_emulate_ops *ops,
2235 struct tss_segment_16 *tss)
2237 struct decode_cache *c = &ctxt->decode;
2240 tss->flag = ctxt->eflags;
2241 tss->ax = c->regs[VCPU_REGS_RAX];
2242 tss->cx = c->regs[VCPU_REGS_RCX];
2243 tss->dx = c->regs[VCPU_REGS_RDX];
2244 tss->bx = c->regs[VCPU_REGS_RBX];
2245 tss->sp = c->regs[VCPU_REGS_RSP];
2246 tss->bp = c->regs[VCPU_REGS_RBP];
2247 tss->si = c->regs[VCPU_REGS_RSI];
2248 tss->di = c->regs[VCPU_REGS_RDI];
2250 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2251 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2252 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2253 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2254 tss->ldt = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2257 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2258 struct x86_emulate_ops *ops,
2259 struct tss_segment_16 *tss)
2261 struct decode_cache *c = &ctxt->decode;
2265 ctxt->eflags = tss->flag | 2;
2266 c->regs[VCPU_REGS_RAX] = tss->ax;
2267 c->regs[VCPU_REGS_RCX] = tss->cx;
2268 c->regs[VCPU_REGS_RDX] = tss->dx;
2269 c->regs[VCPU_REGS_RBX] = tss->bx;
2270 c->regs[VCPU_REGS_RSP] = tss->sp;
2271 c->regs[VCPU_REGS_RBP] = tss->bp;
2272 c->regs[VCPU_REGS_RSI] = tss->si;
2273 c->regs[VCPU_REGS_RDI] = tss->di;
2276 * SDM says that segment selectors are loaded before segment
2279 ops->set_segment_selector(tss->ldt, VCPU_SREG_LDTR, ctxt->vcpu);
2280 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2281 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2282 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2283 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2286 * Now load segment descriptors. If fault happenes at this stage
2287 * it is handled in a context of new task
2289 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
2290 if (ret != X86EMUL_CONTINUE)
2292 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2293 if (ret != X86EMUL_CONTINUE)
2295 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2296 if (ret != X86EMUL_CONTINUE)
2298 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2299 if (ret != X86EMUL_CONTINUE)
2301 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2302 if (ret != X86EMUL_CONTINUE)
2305 return X86EMUL_CONTINUE;
2308 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2309 struct x86_emulate_ops *ops,
2310 u16 tss_selector, u16 old_tss_sel,
2311 ulong old_tss_base, struct desc_struct *new_desc)
2313 struct tss_segment_16 tss_seg;
2315 u32 err, new_tss_base = get_desc_base(new_desc);
2317 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2319 if (ret == X86EMUL_PROPAGATE_FAULT) {
2320 /* FIXME: need to provide precise fault address */
2321 emulate_pf(ctxt, old_tss_base, err);
2325 save_state_to_tss16(ctxt, ops, &tss_seg);
2327 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2329 if (ret == X86EMUL_PROPAGATE_FAULT) {
2330 /* FIXME: need to provide precise fault address */
2331 emulate_pf(ctxt, old_tss_base, err);
2335 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2337 if (ret == X86EMUL_PROPAGATE_FAULT) {
2338 /* FIXME: need to provide precise fault address */
2339 emulate_pf(ctxt, new_tss_base, err);
2343 if (old_tss_sel != 0xffff) {
2344 tss_seg.prev_task_link = old_tss_sel;
2346 ret = ops->write_std(new_tss_base,
2347 &tss_seg.prev_task_link,
2348 sizeof tss_seg.prev_task_link,
2350 if (ret == X86EMUL_PROPAGATE_FAULT) {
2351 /* FIXME: need to provide precise fault address */
2352 emulate_pf(ctxt, new_tss_base, err);
2357 return load_state_from_tss16(ctxt, ops, &tss_seg);
2360 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2361 struct x86_emulate_ops *ops,
2362 struct tss_segment_32 *tss)
2364 struct decode_cache *c = &ctxt->decode;
2366 tss->cr3 = ops->get_cr(3, ctxt->vcpu);
2368 tss->eflags = ctxt->eflags;
2369 tss->eax = c->regs[VCPU_REGS_RAX];
2370 tss->ecx = c->regs[VCPU_REGS_RCX];
2371 tss->edx = c->regs[VCPU_REGS_RDX];
2372 tss->ebx = c->regs[VCPU_REGS_RBX];
2373 tss->esp = c->regs[VCPU_REGS_RSP];
2374 tss->ebp = c->regs[VCPU_REGS_RBP];
2375 tss->esi = c->regs[VCPU_REGS_RSI];
2376 tss->edi = c->regs[VCPU_REGS_RDI];
2378 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2379 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2380 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2381 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2382 tss->fs = ops->get_segment_selector(VCPU_SREG_FS, ctxt->vcpu);
2383 tss->gs = ops->get_segment_selector(VCPU_SREG_GS, ctxt->vcpu);
2384 tss->ldt_selector = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2387 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2388 struct x86_emulate_ops *ops,
2389 struct tss_segment_32 *tss)
2391 struct decode_cache *c = &ctxt->decode;
2394 if (ops->set_cr(3, tss->cr3, ctxt->vcpu)) {
2395 emulate_gp(ctxt, 0);
2396 return X86EMUL_PROPAGATE_FAULT;
2399 ctxt->eflags = tss->eflags | 2;
2400 c->regs[VCPU_REGS_RAX] = tss->eax;
2401 c->regs[VCPU_REGS_RCX] = tss->ecx;
2402 c->regs[VCPU_REGS_RDX] = tss->edx;
2403 c->regs[VCPU_REGS_RBX] = tss->ebx;
2404 c->regs[VCPU_REGS_RSP] = tss->esp;
2405 c->regs[VCPU_REGS_RBP] = tss->ebp;
2406 c->regs[VCPU_REGS_RSI] = tss->esi;
2407 c->regs[VCPU_REGS_RDI] = tss->edi;
2410 * SDM says that segment selectors are loaded before segment
2413 ops->set_segment_selector(tss->ldt_selector, VCPU_SREG_LDTR, ctxt->vcpu);
2414 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2415 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2416 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2417 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2418 ops->set_segment_selector(tss->fs, VCPU_SREG_FS, ctxt->vcpu);
2419 ops->set_segment_selector(tss->gs, VCPU_SREG_GS, ctxt->vcpu);
2422 * Now load segment descriptors. If fault happenes at this stage
2423 * it is handled in a context of new task
2425 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2426 if (ret != X86EMUL_CONTINUE)
2428 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2429 if (ret != X86EMUL_CONTINUE)
2431 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2432 if (ret != X86EMUL_CONTINUE)
2434 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2435 if (ret != X86EMUL_CONTINUE)
2437 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2438 if (ret != X86EMUL_CONTINUE)
2440 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2441 if (ret != X86EMUL_CONTINUE)
2443 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2444 if (ret != X86EMUL_CONTINUE)
2447 return X86EMUL_CONTINUE;
2450 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2451 struct x86_emulate_ops *ops,
2452 u16 tss_selector, u16 old_tss_sel,
2453 ulong old_tss_base, struct desc_struct *new_desc)
2455 struct tss_segment_32 tss_seg;
2457 u32 err, new_tss_base = get_desc_base(new_desc);
2459 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2461 if (ret == X86EMUL_PROPAGATE_FAULT) {
2462 /* FIXME: need to provide precise fault address */
2463 emulate_pf(ctxt, old_tss_base, err);
2467 save_state_to_tss32(ctxt, ops, &tss_seg);
2469 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2471 if (ret == X86EMUL_PROPAGATE_FAULT) {
2472 /* FIXME: need to provide precise fault address */
2473 emulate_pf(ctxt, old_tss_base, err);
2477 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2479 if (ret == X86EMUL_PROPAGATE_FAULT) {
2480 /* FIXME: need to provide precise fault address */
2481 emulate_pf(ctxt, new_tss_base, err);
2485 if (old_tss_sel != 0xffff) {
2486 tss_seg.prev_task_link = old_tss_sel;
2488 ret = ops->write_std(new_tss_base,
2489 &tss_seg.prev_task_link,
2490 sizeof tss_seg.prev_task_link,
2492 if (ret == X86EMUL_PROPAGATE_FAULT) {
2493 /* FIXME: need to provide precise fault address */
2494 emulate_pf(ctxt, new_tss_base, err);
2499 return load_state_from_tss32(ctxt, ops, &tss_seg);
2502 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2503 struct x86_emulate_ops *ops,
2504 u16 tss_selector, int reason,
2505 bool has_error_code, u32 error_code)
2507 struct desc_struct curr_tss_desc, next_tss_desc;
2509 u16 old_tss_sel = ops->get_segment_selector(VCPU_SREG_TR, ctxt->vcpu);
2510 ulong old_tss_base =
2511 ops->get_cached_segment_base(VCPU_SREG_TR, ctxt->vcpu);
2514 /* FIXME: old_tss_base == ~0 ? */
2516 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2517 if (ret != X86EMUL_CONTINUE)
2519 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2520 if (ret != X86EMUL_CONTINUE)
2523 /* FIXME: check that next_tss_desc is tss */
2525 if (reason != TASK_SWITCH_IRET) {
2526 if ((tss_selector & 3) > next_tss_desc.dpl ||
2527 ops->cpl(ctxt->vcpu) > next_tss_desc.dpl) {
2528 emulate_gp(ctxt, 0);
2529 return X86EMUL_PROPAGATE_FAULT;
2533 desc_limit = desc_limit_scaled(&next_tss_desc);
2534 if (!next_tss_desc.p ||
2535 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2536 desc_limit < 0x2b)) {
2537 emulate_ts(ctxt, tss_selector & 0xfffc);
2538 return X86EMUL_PROPAGATE_FAULT;
2541 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2542 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2543 write_segment_descriptor(ctxt, ops, old_tss_sel,
2547 if (reason == TASK_SWITCH_IRET)
2548 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2550 /* set back link to prev task only if NT bit is set in eflags
2551 note that old_tss_sel is not used afetr this point */
2552 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2553 old_tss_sel = 0xffff;
2555 if (next_tss_desc.type & 8)
2556 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2557 old_tss_base, &next_tss_desc);
2559 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2560 old_tss_base, &next_tss_desc);
2561 if (ret != X86EMUL_CONTINUE)
2564 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2565 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2567 if (reason != TASK_SWITCH_IRET) {
2568 next_tss_desc.type |= (1 << 1); /* set busy flag */
2569 write_segment_descriptor(ctxt, ops, tss_selector,
2573 ops->set_cr(0, ops->get_cr(0, ctxt->vcpu) | X86_CR0_TS, ctxt->vcpu);
2574 ops->set_cached_descriptor(&next_tss_desc, VCPU_SREG_TR, ctxt->vcpu);
2575 ops->set_segment_selector(tss_selector, VCPU_SREG_TR, ctxt->vcpu);
2577 if (has_error_code) {
2578 struct decode_cache *c = &ctxt->decode;
2580 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2582 c->src.val = (unsigned long) error_code;
2583 emulate_push(ctxt, ops);
2589 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2590 u16 tss_selector, int reason,
2591 bool has_error_code, u32 error_code)
2593 struct x86_emulate_ops *ops = ctxt->ops;
2594 struct decode_cache *c = &ctxt->decode;
2598 c->dst.type = OP_NONE;
2600 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2601 has_error_code, error_code);
2603 if (rc == X86EMUL_CONTINUE) {
2604 rc = writeback(ctxt, ops);
2605 if (rc == X86EMUL_CONTINUE)
2609 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
2612 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned long base,
2613 int reg, struct operand *op)
2615 struct decode_cache *c = &ctxt->decode;
2616 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2618 register_address_increment(c, &c->regs[reg], df * op->bytes);
2619 op->ptr = (unsigned long *)register_address(c, base, c->regs[reg]);
2623 x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
2625 struct x86_emulate_ops *ops = ctxt->ops;
2627 struct decode_cache *c = &ctxt->decode;
2628 int rc = X86EMUL_CONTINUE;
2629 int saved_dst_type = c->dst.type;
2631 ctxt->decode.mem_read.pos = 0;
2633 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
2638 /* LOCK prefix is allowed only with some instructions */
2639 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
2644 /* Privileged instruction can be executed only in CPL=0 */
2645 if ((c->d & Priv) && ops->cpl(ctxt->vcpu)) {
2646 emulate_gp(ctxt, 0);
2650 if (c->rep_prefix && (c->d & String)) {
2651 ctxt->restart = true;
2652 /* All REP prefixes have the same first termination condition */
2653 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
2655 ctxt->restart = false;
2659 /* The second termination condition only applies for REPE
2660 * and REPNE. Test if the repeat string operation prefix is
2661 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
2662 * corresponding termination condition according to:
2663 * - if REPE/REPZ and ZF = 0 then done
2664 * - if REPNE/REPNZ and ZF = 1 then done
2666 if ((c->b == 0xa6) || (c->b == 0xa7) ||
2667 (c->b == 0xae) || (c->b == 0xaf)) {
2668 if ((c->rep_prefix == REPE_PREFIX) &&
2669 ((ctxt->eflags & EFLG_ZF) == 0))
2671 if ((c->rep_prefix == REPNE_PREFIX) &&
2672 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))
2678 if (c->src.type == OP_MEM) {
2679 rc = read_emulated(ctxt, ops, (unsigned long)c->src.ptr,
2680 c->src.valptr, c->src.bytes);
2681 if (rc != X86EMUL_CONTINUE)
2683 c->src.orig_val64 = c->src.val64;
2686 if (c->src2.type == OP_MEM) {
2687 rc = read_emulated(ctxt, ops, (unsigned long)c->src2.ptr,
2688 &c->src2.val, c->src2.bytes);
2689 if (rc != X86EMUL_CONTINUE)
2693 if ((c->d & DstMask) == ImplicitOps)
2697 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
2698 /* optimisation - avoid slow emulated read if Mov */
2699 rc = read_emulated(ctxt, ops, (unsigned long)c->dst.ptr,
2700 &c->dst.val, c->dst.bytes);
2701 if (rc != X86EMUL_CONTINUE)
2704 c->dst.orig_val = c->dst.val;
2714 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
2716 case 0x06: /* push es */
2717 emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
2719 case 0x07: /* pop es */
2720 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
2721 if (rc != X86EMUL_CONTINUE)
2726 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2728 case 0x0e: /* push cs */
2729 emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
2733 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
2735 case 0x16: /* push ss */
2736 emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
2738 case 0x17: /* pop ss */
2739 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
2740 if (rc != X86EMUL_CONTINUE)
2745 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
2747 case 0x1e: /* push ds */
2748 emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
2750 case 0x1f: /* pop ds */
2751 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
2752 if (rc != X86EMUL_CONTINUE)
2757 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
2761 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
2765 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
2769 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2771 case 0x40 ... 0x47: /* inc r16/r32 */
2772 emulate_1op("inc", c->dst, ctxt->eflags);
2774 case 0x48 ... 0x4f: /* dec r16/r32 */
2775 emulate_1op("dec", c->dst, ctxt->eflags);
2777 case 0x50 ... 0x57: /* push reg */
2778 emulate_push(ctxt, ops);
2780 case 0x58 ... 0x5f: /* pop reg */
2782 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
2783 if (rc != X86EMUL_CONTINUE)
2786 case 0x60: /* pusha */
2787 rc = emulate_pusha(ctxt, ops);
2788 if (rc != X86EMUL_CONTINUE)
2791 case 0x61: /* popa */
2792 rc = emulate_popa(ctxt, ops);
2793 if (rc != X86EMUL_CONTINUE)
2796 case 0x63: /* movsxd */
2797 if (ctxt->mode != X86EMUL_MODE_PROT64)
2798 goto cannot_emulate;
2799 c->dst.val = (s32) c->src.val;
2801 case 0x68: /* push imm */
2802 case 0x6a: /* push imm8 */
2803 emulate_push(ctxt, ops);
2805 case 0x6c: /* insb */
2806 case 0x6d: /* insw/insd */
2807 c->dst.bytes = min(c->dst.bytes, 4u);
2808 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2810 emulate_gp(ctxt, 0);
2813 if (!pio_in_emulated(ctxt, ops, c->dst.bytes,
2814 c->regs[VCPU_REGS_RDX], &c->dst.val))
2815 goto done; /* IO is needed, skip writeback */
2817 case 0x6e: /* outsb */
2818 case 0x6f: /* outsw/outsd */
2819 c->src.bytes = min(c->src.bytes, 4u);
2820 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2822 emulate_gp(ctxt, 0);
2825 ops->pio_out_emulated(c->src.bytes, c->regs[VCPU_REGS_RDX],
2826 &c->src.val, 1, ctxt->vcpu);
2828 c->dst.type = OP_NONE; /* nothing to writeback */
2830 case 0x70 ... 0x7f: /* jcc (short) */
2831 if (test_cc(c->b, ctxt->eflags))
2832 jmp_rel(c, c->src.val);
2834 case 0x80 ... 0x83: /* Grp1 */
2835 switch (c->modrm_reg) {
2856 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
2858 case 0x86 ... 0x87: /* xchg */
2860 /* Write back the register source. */
2861 switch (c->dst.bytes) {
2863 *(u8 *) c->src.ptr = (u8) c->dst.val;
2866 *(u16 *) c->src.ptr = (u16) c->dst.val;
2869 *c->src.ptr = (u32) c->dst.val;
2870 break; /* 64b reg: zero-extend */
2872 *c->src.ptr = c->dst.val;
2876 * Write back the memory destination with implicit LOCK
2879 c->dst.val = c->src.val;
2882 case 0x88 ... 0x8b: /* mov */
2884 case 0x8c: /* mov r/m, sreg */
2885 if (c->modrm_reg > VCPU_SREG_GS) {
2889 c->dst.val = ops->get_segment_selector(c->modrm_reg, ctxt->vcpu);
2891 case 0x8d: /* lea r16/r32, m */
2892 c->dst.val = c->modrm_ea;
2894 case 0x8e: { /* mov seg, r/m16 */
2899 if (c->modrm_reg == VCPU_SREG_CS ||
2900 c->modrm_reg > VCPU_SREG_GS) {
2905 if (c->modrm_reg == VCPU_SREG_SS)
2906 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2908 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
2910 c->dst.type = OP_NONE; /* Disable writeback. */
2913 case 0x8f: /* pop (sole member of Grp1a) */
2914 rc = emulate_grp1a(ctxt, ops);
2915 if (rc != X86EMUL_CONTINUE)
2918 case 0x90: /* nop / xchg r8,rax */
2919 if (c->dst.ptr == (unsigned long *)&c->regs[VCPU_REGS_RAX]) {
2920 c->dst.type = OP_NONE; /* nop */
2923 case 0x91 ... 0x97: /* xchg reg,rax */
2924 c->src.type = OP_REG;
2925 c->src.bytes = c->op_bytes;
2926 c->src.ptr = (unsigned long *) &c->regs[VCPU_REGS_RAX];
2927 c->src.val = *(c->src.ptr);
2929 case 0x9c: /* pushf */
2930 c->src.val = (unsigned long) ctxt->eflags;
2931 emulate_push(ctxt, ops);
2933 case 0x9d: /* popf */
2934 c->dst.type = OP_REG;
2935 c->dst.ptr = (unsigned long *) &ctxt->eflags;
2936 c->dst.bytes = c->op_bytes;
2937 rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
2938 if (rc != X86EMUL_CONTINUE)
2941 case 0xa0 ... 0xa3: /* mov */
2942 case 0xa4 ... 0xa5: /* movs */
2944 case 0xa6 ... 0xa7: /* cmps */
2945 c->dst.type = OP_NONE; /* Disable writeback. */
2946 DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.ptr, c->dst.ptr);
2948 case 0xa8 ... 0xa9: /* test ax, imm */
2950 case 0xaa ... 0xab: /* stos */
2951 c->dst.val = c->regs[VCPU_REGS_RAX];
2953 case 0xac ... 0xad: /* lods */
2955 case 0xae ... 0xaf: /* scas */
2956 DPRINTF("Urk! I don't handle SCAS.\n");
2957 goto cannot_emulate;
2958 case 0xb0 ... 0xbf: /* mov r, imm */
2963 case 0xc3: /* ret */
2964 c->dst.type = OP_REG;
2965 c->dst.ptr = &c->eip;
2966 c->dst.bytes = c->op_bytes;
2967 goto pop_instruction;
2968 case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */
2970 c->dst.val = c->src.val;
2972 case 0xcb: /* ret far */
2973 rc = emulate_ret_far(ctxt, ops);
2974 if (rc != X86EMUL_CONTINUE)
2977 case 0xcf: /* iret */
2978 rc = emulate_iret(ctxt, ops);
2980 if (rc != X86EMUL_CONTINUE)
2983 case 0xd0 ... 0xd1: /* Grp2 */
2987 case 0xd2 ... 0xd3: /* Grp2 */
2988 c->src.val = c->regs[VCPU_REGS_RCX];
2991 case 0xe4: /* inb */
2994 case 0xe6: /* outb */
2995 case 0xe7: /* out */
2997 case 0xe8: /* call (near) */ {
2998 long int rel = c->src.val;
2999 c->src.val = (unsigned long) c->eip;
3001 emulate_push(ctxt, ops);
3004 case 0xe9: /* jmp rel */
3006 case 0xea: { /* jmp far */
3009 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
3011 if (load_segment_descriptor(ctxt, ops, sel, VCPU_SREG_CS))
3015 memcpy(&c->eip, c->src.valptr, c->op_bytes);
3019 jmp: /* jmp rel short */
3020 jmp_rel(c, c->src.val);
3021 c->dst.type = OP_NONE; /* Disable writeback. */
3023 case 0xec: /* in al,dx */
3024 case 0xed: /* in (e/r)ax,dx */
3025 c->src.val = c->regs[VCPU_REGS_RDX];
3027 c->dst.bytes = min(c->dst.bytes, 4u);
3028 if (!emulator_io_permited(ctxt, ops, c->src.val, c->dst.bytes)) {
3029 emulate_gp(ctxt, 0);
3032 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
3034 goto done; /* IO is needed */
3036 case 0xee: /* out dx,al */
3037 case 0xef: /* out dx,(e/r)ax */
3038 c->src.val = c->regs[VCPU_REGS_RDX];
3040 c->dst.bytes = min(c->dst.bytes, 4u);
3041 if (!emulator_io_permited(ctxt, ops, c->src.val, c->dst.bytes)) {
3042 emulate_gp(ctxt, 0);
3045 ops->pio_out_emulated(c->dst.bytes, c->src.val, &c->dst.val, 1,
3047 c->dst.type = OP_NONE; /* Disable writeback. */
3049 case 0xf4: /* hlt */
3050 ctxt->vcpu->arch.halt_request = 1;
3052 case 0xf5: /* cmc */
3053 /* complement carry flag from eflags reg */
3054 ctxt->eflags ^= EFLG_CF;
3055 c->dst.type = OP_NONE; /* Disable writeback. */
3057 case 0xf6 ... 0xf7: /* Grp3 */
3058 if (!emulate_grp3(ctxt, ops))
3059 goto cannot_emulate;
3061 case 0xf8: /* clc */
3062 ctxt->eflags &= ~EFLG_CF;
3063 c->dst.type = OP_NONE; /* Disable writeback. */
3065 case 0xfa: /* cli */
3066 if (emulator_bad_iopl(ctxt, ops)) {
3067 emulate_gp(ctxt, 0);
3070 ctxt->eflags &= ~X86_EFLAGS_IF;
3071 c->dst.type = OP_NONE; /* Disable writeback. */
3074 case 0xfb: /* sti */
3075 if (emulator_bad_iopl(ctxt, ops)) {
3076 emulate_gp(ctxt, 0);
3079 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3080 ctxt->eflags |= X86_EFLAGS_IF;
3081 c->dst.type = OP_NONE; /* Disable writeback. */
3084 case 0xfc: /* cld */
3085 ctxt->eflags &= ~EFLG_DF;
3086 c->dst.type = OP_NONE; /* Disable writeback. */
3088 case 0xfd: /* std */
3089 ctxt->eflags |= EFLG_DF;
3090 c->dst.type = OP_NONE; /* Disable writeback. */
3092 case 0xfe: /* Grp4 */
3094 rc = emulate_grp45(ctxt, ops);
3095 if (rc != X86EMUL_CONTINUE)
3098 case 0xff: /* Grp5 */
3099 if (c->modrm_reg == 5)
3103 goto cannot_emulate;
3107 rc = writeback(ctxt, ops);
3108 if (rc != X86EMUL_CONTINUE)
3112 * restore dst type in case the decoding will be reused
3113 * (happens for string instruction )
3115 c->dst.type = saved_dst_type;
3117 if ((c->d & SrcMask) == SrcSI)
3118 string_addr_inc(ctxt, seg_override_base(ctxt, ops, c),
3119 VCPU_REGS_RSI, &c->src);
3121 if ((c->d & DstMask) == DstDI)
3122 string_addr_inc(ctxt, es_base(ctxt, ops), VCPU_REGS_RDI,
3125 if (c->rep_prefix && (c->d & String)) {
3126 struct read_cache *rc = &ctxt->decode.io_read;
3127 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3129 * Re-enter guest when pio read ahead buffer is empty or,
3130 * if it is not used, after each 1024 iteration.
3132 if ((rc->end == 0 && !(c->regs[VCPU_REGS_RCX] & 0x3ff)) ||
3133 (rc->end != 0 && rc->end == rc->pos))
3134 ctxt->restart = false;
3137 * reset read cache here in case string instruction is restared
3140 ctxt->decode.mem_read.end = 0;
3144 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
3148 case 0x01: /* lgdt, lidt, lmsw */
3149 switch (c->modrm_reg) {
3151 unsigned long address;
3153 case 0: /* vmcall */
3154 if (c->modrm_mod != 3 || c->modrm_rm != 1)
3155 goto cannot_emulate;
3157 rc = kvm_fix_hypercall(ctxt->vcpu);
3158 if (rc != X86EMUL_CONTINUE)
3161 /* Let the processor re-execute the fixed hypercall */
3163 /* Disable writeback. */
3164 c->dst.type = OP_NONE;
3167 rc = read_descriptor(ctxt, ops, c->src.ptr,
3168 &size, &address, c->op_bytes);
3169 if (rc != X86EMUL_CONTINUE)
3171 realmode_lgdt(ctxt->vcpu, size, address);
3172 /* Disable writeback. */
3173 c->dst.type = OP_NONE;
3175 case 3: /* lidt/vmmcall */
3176 if (c->modrm_mod == 3) {
3177 switch (c->modrm_rm) {
3179 rc = kvm_fix_hypercall(ctxt->vcpu);
3180 if (rc != X86EMUL_CONTINUE)
3184 goto cannot_emulate;
3187 rc = read_descriptor(ctxt, ops, c->src.ptr,
3190 if (rc != X86EMUL_CONTINUE)
3192 realmode_lidt(ctxt->vcpu, size, address);
3194 /* Disable writeback. */
3195 c->dst.type = OP_NONE;
3199 c->dst.val = ops->get_cr(0, ctxt->vcpu);
3202 ops->set_cr(0, (ops->get_cr(0, ctxt->vcpu) & ~0x0ful) |
3203 (c->src.val & 0x0f), ctxt->vcpu);
3204 c->dst.type = OP_NONE;
3206 case 5: /* not defined */
3210 emulate_invlpg(ctxt->vcpu, c->modrm_ea);
3211 /* Disable writeback. */
3212 c->dst.type = OP_NONE;
3215 goto cannot_emulate;
3218 case 0x05: /* syscall */
3219 rc = emulate_syscall(ctxt, ops);
3220 if (rc != X86EMUL_CONTINUE)
3226 emulate_clts(ctxt->vcpu);
3227 c->dst.type = OP_NONE;
3229 case 0x09: /* wbinvd */
3230 kvm_emulate_wbinvd(ctxt->vcpu);
3231 c->dst.type = OP_NONE;
3233 case 0x08: /* invd */
3234 case 0x0d: /* GrpP (prefetch) */
3235 case 0x18: /* Grp16 (prefetch/nop) */
3236 c->dst.type = OP_NONE;
3238 case 0x20: /* mov cr, reg */
3239 switch (c->modrm_reg) {
3246 c->regs[c->modrm_rm] = ops->get_cr(c->modrm_reg, ctxt->vcpu);
3247 c->dst.type = OP_NONE; /* no writeback */
3249 case 0x21: /* mov from dr to reg */
3250 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
3251 (c->modrm_reg == 4 || c->modrm_reg == 5)) {
3255 ops->get_dr(c->modrm_reg, &c->regs[c->modrm_rm], ctxt->vcpu);
3256 c->dst.type = OP_NONE; /* no writeback */
3258 case 0x22: /* mov reg, cr */
3259 if (ops->set_cr(c->modrm_reg, c->modrm_val, ctxt->vcpu)) {
3260 emulate_gp(ctxt, 0);
3263 c->dst.type = OP_NONE;
3265 case 0x23: /* mov from reg to dr */
3266 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
3267 (c->modrm_reg == 4 || c->modrm_reg == 5)) {
3272 if (ops->set_dr(c->modrm_reg, c->regs[c->modrm_rm] &
3273 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
3274 ~0ULL : ~0U), ctxt->vcpu) < 0) {
3275 /* #UD condition is already handled by the code above */
3276 emulate_gp(ctxt, 0);
3280 c->dst.type = OP_NONE; /* no writeback */
3284 msr_data = (u32)c->regs[VCPU_REGS_RAX]
3285 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
3286 if (ops->set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) {
3287 emulate_gp(ctxt, 0);
3290 rc = X86EMUL_CONTINUE;
3291 c->dst.type = OP_NONE;
3295 if (ops->get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) {
3296 emulate_gp(ctxt, 0);
3299 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
3300 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
3302 rc = X86EMUL_CONTINUE;
3303 c->dst.type = OP_NONE;
3305 case 0x34: /* sysenter */
3306 rc = emulate_sysenter(ctxt, ops);
3307 if (rc != X86EMUL_CONTINUE)
3312 case 0x35: /* sysexit */
3313 rc = emulate_sysexit(ctxt, ops);
3314 if (rc != X86EMUL_CONTINUE)
3319 case 0x40 ... 0x4f: /* cmov */
3320 c->dst.val = c->dst.orig_val = c->src.val;
3321 if (!test_cc(c->b, ctxt->eflags))
3322 c->dst.type = OP_NONE; /* no writeback */
3324 case 0x80 ... 0x8f: /* jnz rel, etc*/
3325 if (test_cc(c->b, ctxt->eflags))
3326 jmp_rel(c, c->src.val);
3327 c->dst.type = OP_NONE;
3329 case 0xa0: /* push fs */
3330 emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
3332 case 0xa1: /* pop fs */
3333 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
3334 if (rc != X86EMUL_CONTINUE)
3339 c->dst.type = OP_NONE;
3340 /* only subword offset */
3341 c->src.val &= (c->dst.bytes << 3) - 1;
3342 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
3344 case 0xa4: /* shld imm8, r, r/m */
3345 case 0xa5: /* shld cl, r, r/m */
3346 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
3348 case 0xa8: /* push gs */
3349 emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
3351 case 0xa9: /* pop gs */
3352 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
3353 if (rc != X86EMUL_CONTINUE)
3358 /* only subword offset */
3359 c->src.val &= (c->dst.bytes << 3) - 1;
3360 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
3362 case 0xac: /* shrd imm8, r, r/m */
3363 case 0xad: /* shrd cl, r, r/m */
3364 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
3366 case 0xae: /* clflush */
3368 case 0xb0 ... 0xb1: /* cmpxchg */
3370 * Save real source value, then compare EAX against
3373 c->src.orig_val = c->src.val;
3374 c->src.val = c->regs[VCPU_REGS_RAX];
3375 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
3376 if (ctxt->eflags & EFLG_ZF) {
3377 /* Success: write back to memory. */
3378 c->dst.val = c->src.orig_val;
3380 /* Failure: write the value we saw to EAX. */
3381 c->dst.type = OP_REG;
3382 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
3387 /* only subword offset */
3388 c->src.val &= (c->dst.bytes << 3) - 1;
3389 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
3391 case 0xb6 ... 0xb7: /* movzx */
3392 c->dst.bytes = c->op_bytes;
3393 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
3396 case 0xba: /* Grp8 */
3397 switch (c->modrm_reg & 3) {
3410 /* only subword offset */
3411 c->src.val &= (c->dst.bytes << 3) - 1;
3412 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
3414 case 0xbe ... 0xbf: /* movsx */
3415 c->dst.bytes = c->op_bytes;
3416 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
3419 case 0xc3: /* movnti */
3420 c->dst.bytes = c->op_bytes;
3421 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
3424 case 0xc7: /* Grp9 (cmpxchg8b) */
3425 rc = emulate_grp9(ctxt, ops);
3426 if (rc != X86EMUL_CONTINUE)
3430 goto cannot_emulate;
3435 DPRINTF("Cannot emulate %02x\n", c->b);