1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affilates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
26 #include <public/xen.h>
27 #define DPRINTF(_f, _a ...) printf(_f , ## _a)
29 #include <linux/kvm_host.h>
30 #include "kvm_cache_regs.h"
31 #define DPRINTF(x...) do {} while (0)
33 #include <linux/module.h>
34 #include <asm/kvm_emulate.h>
40 * Opcode effective-address decode tables.
41 * Note that we only emulate instructions that have at least one memory
42 * operand (excluding implicit stack references). We assume that stack
43 * references and instruction fetches will never occur in special memory
44 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
48 /* Operand sizes: 8-bit operands or specified/overridden size. */
49 #define ByteOp (1<<0) /* 8-bit operands. */
50 /* Destination operand type. */
51 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
52 #define DstReg (2<<1) /* Register operand. */
53 #define DstMem (3<<1) /* Memory operand. */
54 #define DstAcc (4<<1) /* Destination Accumulator */
55 #define DstDI (5<<1) /* Destination is in ES:(E)DI */
56 #define DstMem64 (6<<1) /* 64bit memory operand */
57 #define DstMask (7<<1)
58 /* Source operand type. */
59 #define SrcNone (0<<4) /* No source operand. */
60 #define SrcImplicit (0<<4) /* Source operand is implicit in the opcode. */
61 #define SrcReg (1<<4) /* Register operand. */
62 #define SrcMem (2<<4) /* Memory operand. */
63 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
64 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
65 #define SrcImm (5<<4) /* Immediate operand. */
66 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
67 #define SrcOne (7<<4) /* Implied '1' */
68 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
69 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
70 #define SrcSI (0xa<<4) /* Source is in the DS:RSI */
71 #define SrcImmFAddr (0xb<<4) /* Source is immediate far address */
72 #define SrcMemFAddr (0xc<<4) /* Source is far address in memory */
73 #define SrcAcc (0xd<<4) /* Source Accumulator */
74 #define SrcMask (0xf<<4)
75 /* Generic ModRM decode. */
77 /* Destination is only written; never read. */
80 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
81 #define String (1<<12) /* String instruction (rep capable) */
82 #define Stack (1<<13) /* Stack instruction (push/pop) */
83 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
84 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
86 #define Undefined (1<<25) /* No Such Instruction */
87 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
88 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
90 /* Source 2 operand type */
91 #define Src2None (0<<29)
92 #define Src2CL (1<<29)
93 #define Src2ImmByte (2<<29)
94 #define Src2One (3<<29)
95 #define Src2Mask (7<<29)
98 #define X3(x) X2(x), x
99 #define X4(x) X2(x), X2(x)
100 #define X5(x) X4(x), x
101 #define X6(x) X4(x), X2(x)
102 #define X7(x) X4(x), X3(x)
103 #define X8(x) X4(x), X4(x)
104 #define X16(x) X8(x), X8(x)
109 int (*execute)(struct x86_emulate_ctxt *ctxt);
110 struct opcode *group;
111 struct group_dual *gdual;
116 struct opcode mod012[8];
117 struct opcode mod3[8];
120 #define D(_y) { .flags = (_y) }
122 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
123 #define GD(_f, _g) { .flags = ((_f) | Group | GroupDual), .u.gdual = (_g) }
124 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
126 static struct opcode group1[] = {
130 static struct opcode group1A[] = {
131 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
134 static struct opcode group3[] = {
135 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
136 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
140 static struct opcode group4[] = {
141 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
145 static struct opcode group5[] = {
146 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
147 D(SrcMem | ModRM | Stack), N,
148 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
149 D(SrcMem | ModRM | Stack), N,
152 static struct group_dual group7 = { {
153 N, N, D(ModRM | SrcMem | Priv), D(ModRM | SrcMem | Priv),
154 D(SrcNone | ModRM | DstMem | Mov), N,
155 D(SrcMem16 | ModRM | Mov | Priv), D(SrcMem | ModRM | ByteOp | Priv),
157 D(SrcNone | ModRM | Priv), N, N, D(SrcNone | ModRM | Priv),
158 D(SrcNone | ModRM | DstMem | Mov), N,
159 D(SrcMem16 | ModRM | Mov | Priv), N,
162 static struct opcode group8[] = {
164 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
165 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
168 static struct group_dual group9 = { {
169 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
171 N, N, N, N, N, N, N, N,
174 static struct opcode opcode_table[256] = {
176 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
177 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
178 D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
179 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
181 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
182 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
183 D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
184 D(ImplicitOps | Stack | No64), N,
186 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
187 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
188 D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
189 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
191 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
192 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
193 D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
194 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
196 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
197 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
198 D(ByteOp | DstAcc | SrcImmByte), D(DstAcc | SrcImm), N, N,
200 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
201 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
202 D(ByteOp | DstAcc | SrcImmByte), D(DstAcc | SrcImm), N, N,
204 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
205 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
206 D(ByteOp | DstAcc | SrcImmByte), D(DstAcc | SrcImm), N, N,
208 D(ByteOp | DstMem | SrcReg | ModRM), D(DstMem | SrcReg | ModRM),
209 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
210 D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
215 X8(D(SrcReg | Stack)),
217 X8(D(DstReg | Stack)),
219 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
220 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
223 D(SrcImm | Mov | Stack), N, D(SrcImmByte | Mov | Stack), N,
224 D(DstDI | ByteOp | Mov | String), D(DstDI | Mov | String), /* insb, insw/insd */
225 D(SrcSI | ByteOp | ImplicitOps | String), D(SrcSI | ImplicitOps | String), /* outsb, outsw/outsd */
229 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
230 G(DstMem | SrcImm | ModRM | Group, group1),
231 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
232 G(DstMem | SrcImmByte | ModRM | Group, group1),
233 D(ByteOp | DstMem | SrcReg | ModRM), D(DstMem | SrcReg | ModRM),
234 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
236 D(ByteOp | DstMem | SrcReg | ModRM | Mov), D(DstMem | SrcReg | ModRM | Mov),
237 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem | ModRM | Mov),
238 D(DstMem | SrcNone | ModRM | Mov), D(ModRM | DstReg),
239 D(ImplicitOps | SrcMem16 | ModRM), G(0, group1A),
241 D(DstReg), D(DstReg), D(DstReg), D(DstReg), D(DstReg), D(DstReg), D(DstReg), D(DstReg),
243 N, N, D(SrcImmFAddr | No64), N,
244 D(ImplicitOps | Stack), D(ImplicitOps | Stack), N, N,
246 D(ByteOp | DstAcc | SrcMem | Mov | MemAbs), D(DstAcc | SrcMem | Mov | MemAbs),
247 D(ByteOp | DstMem | SrcAcc | Mov | MemAbs), D(DstMem | SrcAcc | Mov | MemAbs),
248 D(ByteOp | SrcSI | DstDI | Mov | String), D(SrcSI | DstDI | Mov | String),
249 D(ByteOp | SrcSI | DstDI | String), D(SrcSI | DstDI | String),
251 D(DstAcc | SrcImmByte | ByteOp), D(DstAcc | SrcImm), D(ByteOp | DstDI | Mov | String), D(DstDI | Mov | String),
252 D(ByteOp | SrcSI | DstAcc | Mov | String), D(SrcSI | DstAcc | Mov | String),
253 D(ByteOp | DstDI | String), D(DstDI | String),
255 X8(D(ByteOp | DstReg | SrcImm | Mov)),
257 X8(D(DstReg | SrcImm | Mov)),
259 D(ByteOp | DstMem | SrcImm | ModRM), D(DstMem | SrcImmByte | ModRM),
260 N, D(ImplicitOps | Stack), N, N,
261 D(ByteOp | DstMem | SrcImm | ModRM | Mov), D(DstMem | SrcImm | ModRM | Mov),
263 N, N, N, D(ImplicitOps | Stack),
264 D(ImplicitOps), D(SrcImmByte), D(ImplicitOps | No64), D(ImplicitOps),
266 D(ByteOp | DstMem | SrcImplicit | ModRM), D(DstMem | SrcImplicit | ModRM),
267 D(ByteOp | DstMem | SrcImplicit | ModRM), D(DstMem | SrcImplicit | ModRM),
270 N, N, N, N, N, N, N, N,
273 D(ByteOp | SrcImmUByte | DstAcc), D(SrcImmUByte | DstAcc),
274 D(ByteOp | SrcImmUByte | DstAcc), D(SrcImmUByte | DstAcc),
276 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
277 D(SrcImmFAddr | No64), D(SrcImmByte | ImplicitOps),
278 D(SrcNone | ByteOp | DstAcc), D(SrcNone | DstAcc),
279 D(SrcNone | ByteOp | DstAcc), D(SrcNone | DstAcc),
282 D(ImplicitOps | Priv), D(ImplicitOps), G(ByteOp, group3), G(0, group3),
284 D(ImplicitOps), N, D(ImplicitOps), D(ImplicitOps),
285 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
288 static struct opcode twobyte_table[256] = {
290 N, GD(0, &group7), N, N,
291 N, D(ImplicitOps), D(ImplicitOps | Priv), N,
292 D(ImplicitOps | Priv), D(ImplicitOps | Priv), N, N,
293 N, D(ImplicitOps | ModRM), N, N,
295 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
297 D(ModRM | ImplicitOps | Priv), D(ModRM | Priv),
298 D(ModRM | ImplicitOps | Priv), D(ModRM | Priv),
300 N, N, N, N, N, N, N, N,
302 D(ImplicitOps | Priv), N, D(ImplicitOps | Priv), N,
303 D(ImplicitOps), D(ImplicitOps | Priv), N, N,
304 N, N, N, N, N, N, N, N,
306 X16(D(DstReg | SrcMem | ModRM | Mov)),
308 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
310 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
312 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
316 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
318 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
319 N, D(DstMem | SrcReg | ModRM | BitOp),
320 D(DstMem | SrcReg | Src2ImmByte | ModRM),
321 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
323 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
324 N, D(DstMem | SrcReg | ModRM | BitOp | Lock),
325 D(DstMem | SrcReg | Src2ImmByte | ModRM),
326 D(DstMem | SrcReg | Src2CL | ModRM),
329 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
330 N, D(DstMem | SrcReg | ModRM | BitOp | Lock),
331 N, N, D(ByteOp | DstReg | SrcMem | ModRM | Mov),
332 D(DstReg | SrcMem16 | ModRM | Mov),
335 G(0, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
336 N, N, D(ByteOp | DstReg | SrcMem | ModRM | Mov),
337 D(DstReg | SrcMem16 | ModRM | Mov),
339 N, N, N, D(DstMem | SrcReg | ModRM | Mov),
340 N, N, N, GD(0, &group9),
341 N, N, N, N, N, N, N, N,
343 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
345 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
347 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
356 /* EFLAGS bit definitions. */
357 #define EFLG_ID (1<<21)
358 #define EFLG_VIP (1<<20)
359 #define EFLG_VIF (1<<19)
360 #define EFLG_AC (1<<18)
361 #define EFLG_VM (1<<17)
362 #define EFLG_RF (1<<16)
363 #define EFLG_IOPL (3<<12)
364 #define EFLG_NT (1<<14)
365 #define EFLG_OF (1<<11)
366 #define EFLG_DF (1<<10)
367 #define EFLG_IF (1<<9)
368 #define EFLG_TF (1<<8)
369 #define EFLG_SF (1<<7)
370 #define EFLG_ZF (1<<6)
371 #define EFLG_AF (1<<4)
372 #define EFLG_PF (1<<2)
373 #define EFLG_CF (1<<0)
375 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
376 #define EFLG_RESERVED_ONE_MASK 2
379 * Instruction emulation:
380 * Most instructions are emulated directly via a fragment of inline assembly
381 * code. This allows us to save/restore EFLAGS and thus very easily pick up
382 * any modified flags.
385 #if defined(CONFIG_X86_64)
386 #define _LO32 "k" /* force 32-bit operand */
387 #define _STK "%%rsp" /* stack pointer */
388 #elif defined(__i386__)
389 #define _LO32 "" /* force 32-bit operand */
390 #define _STK "%%esp" /* stack pointer */
394 * These EFLAGS bits are restored from saved value during emulation, and
395 * any changes are written back to the saved value after emulation.
397 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
399 /* Before executing instruction: restore necessary bits in EFLAGS. */
400 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
401 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
402 "movl %"_sav",%"_LO32 _tmp"; " \
405 "movl %"_msk",%"_LO32 _tmp"; " \
406 "andl %"_LO32 _tmp",("_STK"); " \
408 "notl %"_LO32 _tmp"; " \
409 "andl %"_LO32 _tmp",("_STK"); " \
410 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
412 "orl %"_LO32 _tmp",("_STK"); " \
416 /* After executing instruction: write-back necessary bits in EFLAGS. */
417 #define _POST_EFLAGS(_sav, _msk, _tmp) \
418 /* _sav |= EFLAGS & _msk; */ \
421 "andl %"_msk",%"_LO32 _tmp"; " \
422 "orl %"_LO32 _tmp",%"_sav"; "
430 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix) \
432 __asm__ __volatile__ ( \
433 _PRE_EFLAGS("0", "4", "2") \
434 _op _suffix " %"_x"3,%1; " \
435 _POST_EFLAGS("0", "4", "2") \
436 : "=m" (_eflags), "=m" ((_dst).val), \
438 : _y ((_src).val), "i" (EFLAGS_MASK)); \
442 /* Raw emulation: instruction has two explicit operands. */
443 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
445 unsigned long _tmp; \
447 switch ((_dst).bytes) { \
449 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w"); \
452 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l"); \
455 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q")); \
460 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
462 unsigned long _tmp; \
463 switch ((_dst).bytes) { \
465 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b"); \
468 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
469 _wx, _wy, _lx, _ly, _qx, _qy); \
474 /* Source operand is byte-sized and may be restricted to just %cl. */
475 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
476 __emulate_2op(_op, _src, _dst, _eflags, \
477 "b", "c", "b", "c", "b", "c", "b", "c")
479 /* Source operand is byte, word, long or quad sized. */
480 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
481 __emulate_2op(_op, _src, _dst, _eflags, \
482 "b", "q", "w", "r", _LO32, "r", "", "r")
484 /* Source operand is word, long or quad sized. */
485 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
486 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
487 "w", "r", _LO32, "r", "", "r")
489 /* Instruction has three operands and one operand is stored in ECX register */
490 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
492 unsigned long _tmp; \
493 _type _clv = (_cl).val; \
494 _type _srcv = (_src).val; \
495 _type _dstv = (_dst).val; \
497 __asm__ __volatile__ ( \
498 _PRE_EFLAGS("0", "5", "2") \
499 _op _suffix " %4,%1 \n" \
500 _POST_EFLAGS("0", "5", "2") \
501 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
502 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
505 (_cl).val = (unsigned long) _clv; \
506 (_src).val = (unsigned long) _srcv; \
507 (_dst).val = (unsigned long) _dstv; \
510 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
512 switch ((_dst).bytes) { \
514 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
515 "w", unsigned short); \
518 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
519 "l", unsigned int); \
522 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
523 "q", unsigned long)); \
528 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
530 unsigned long _tmp; \
532 __asm__ __volatile__ ( \
533 _PRE_EFLAGS("0", "3", "2") \
534 _op _suffix " %1; " \
535 _POST_EFLAGS("0", "3", "2") \
536 : "=m" (_eflags), "+m" ((_dst).val), \
538 : "i" (EFLAGS_MASK)); \
541 /* Instruction has only one explicit operand (no source operand). */
542 #define emulate_1op(_op, _dst, _eflags) \
544 switch ((_dst).bytes) { \
545 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
546 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
547 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
548 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
552 /* Fetch next part of the instruction being emulated. */
553 #define insn_fetch(_type, _size, _eip) \
554 ({ unsigned long _x; \
555 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
556 if (rc != X86EMUL_CONTINUE) \
562 #define insn_fetch_arr(_arr, _size, _eip) \
563 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
564 if (rc != X86EMUL_CONTINUE) \
569 static inline unsigned long ad_mask(struct decode_cache *c)
571 return (1UL << (c->ad_bytes << 3)) - 1;
574 /* Access/update address held in a register, based on addressing mode. */
575 static inline unsigned long
576 address_mask(struct decode_cache *c, unsigned long reg)
578 if (c->ad_bytes == sizeof(unsigned long))
581 return reg & ad_mask(c);
584 static inline unsigned long
585 register_address(struct decode_cache *c, unsigned long base, unsigned long reg)
587 return base + address_mask(c, reg);
591 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
593 if (c->ad_bytes == sizeof(unsigned long))
596 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
599 static inline void jmp_rel(struct decode_cache *c, int rel)
601 register_address_increment(c, &c->eip, rel);
604 static void set_seg_override(struct decode_cache *c, int seg)
606 c->has_seg_override = true;
607 c->seg_override = seg;
610 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
611 struct x86_emulate_ops *ops, int seg)
613 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
616 return ops->get_cached_segment_base(seg, ctxt->vcpu);
619 static unsigned long seg_override_base(struct x86_emulate_ctxt *ctxt,
620 struct x86_emulate_ops *ops,
621 struct decode_cache *c)
623 if (!c->has_seg_override)
626 return seg_base(ctxt, ops, c->seg_override);
629 static unsigned long es_base(struct x86_emulate_ctxt *ctxt,
630 struct x86_emulate_ops *ops)
632 return seg_base(ctxt, ops, VCPU_SREG_ES);
635 static unsigned long ss_base(struct x86_emulate_ctxt *ctxt,
636 struct x86_emulate_ops *ops)
638 return seg_base(ctxt, ops, VCPU_SREG_SS);
641 static void emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
642 u32 error, bool valid)
644 ctxt->exception = vec;
645 ctxt->error_code = error;
646 ctxt->error_code_valid = valid;
647 ctxt->restart = false;
650 static void emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
652 emulate_exception(ctxt, GP_VECTOR, err, true);
655 static void emulate_pf(struct x86_emulate_ctxt *ctxt, unsigned long addr,
659 emulate_exception(ctxt, PF_VECTOR, err, true);
662 static void emulate_ud(struct x86_emulate_ctxt *ctxt)
664 emulate_exception(ctxt, UD_VECTOR, 0, false);
667 static void emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
669 emulate_exception(ctxt, TS_VECTOR, err, true);
672 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
673 struct x86_emulate_ops *ops,
674 unsigned long eip, u8 *dest)
676 struct fetch_cache *fc = &ctxt->decode.fetch;
680 if (eip == fc->end) {
681 cur_size = fc->end - fc->start;
682 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
683 rc = ops->fetch(ctxt->cs_base + eip, fc->data + cur_size,
684 size, ctxt->vcpu, NULL);
685 if (rc != X86EMUL_CONTINUE)
689 *dest = fc->data[eip - fc->start];
690 return X86EMUL_CONTINUE;
693 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
694 struct x86_emulate_ops *ops,
695 unsigned long eip, void *dest, unsigned size)
699 /* x86 instructions are limited to 15 bytes. */
700 if (eip + size - ctxt->eip > 15)
701 return X86EMUL_UNHANDLEABLE;
703 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
704 if (rc != X86EMUL_CONTINUE)
707 return X86EMUL_CONTINUE;
711 * Given the 'reg' portion of a ModRM byte, and a register block, return a
712 * pointer into the block that addresses the relevant register.
713 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
715 static void *decode_register(u8 modrm_reg, unsigned long *regs,
720 p = ®s[modrm_reg];
721 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
722 p = (unsigned char *)®s[modrm_reg & 3] + 1;
726 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
727 struct x86_emulate_ops *ops,
729 u16 *size, unsigned long *address, int op_bytes)
736 rc = ops->read_std((unsigned long)ptr, (unsigned long *)size, 2,
738 if (rc != X86EMUL_CONTINUE)
740 rc = ops->read_std((unsigned long)ptr + 2, address, op_bytes,
745 static int test_cc(unsigned int condition, unsigned int flags)
749 switch ((condition & 15) >> 1) {
751 rc |= (flags & EFLG_OF);
753 case 1: /* b/c/nae */
754 rc |= (flags & EFLG_CF);
757 rc |= (flags & EFLG_ZF);
760 rc |= (flags & (EFLG_CF|EFLG_ZF));
763 rc |= (flags & EFLG_SF);
766 rc |= (flags & EFLG_PF);
769 rc |= (flags & EFLG_ZF);
772 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
776 /* Odd condition identifiers (lsb == 1) have inverted sense. */
777 return (!!rc ^ (condition & 1));
780 static void decode_register_operand(struct operand *op,
781 struct decode_cache *c,
784 unsigned reg = c->modrm_reg;
785 int highbyte_regs = c->rex_prefix == 0;
788 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
790 if ((c->d & ByteOp) && !inhibit_bytereg) {
791 op->ptr = decode_register(reg, c->regs, highbyte_regs);
792 op->val = *(u8 *)op->ptr;
795 op->ptr = decode_register(reg, c->regs, 0);
796 op->bytes = c->op_bytes;
799 op->val = *(u16 *)op->ptr;
802 op->val = *(u32 *)op->ptr;
805 op->val = *(u64 *) op->ptr;
809 op->orig_val = op->val;
812 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
813 struct x86_emulate_ops *ops)
815 struct decode_cache *c = &ctxt->decode;
817 int index_reg = 0, base_reg = 0, scale;
818 int rc = X86EMUL_CONTINUE;
821 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
822 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
823 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
826 c->modrm = insn_fetch(u8, 1, c->eip);
827 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
828 c->modrm_reg |= (c->modrm & 0x38) >> 3;
829 c->modrm_rm |= (c->modrm & 0x07);
833 if (c->modrm_mod == 3) {
834 c->modrm_ptr = decode_register(c->modrm_rm,
835 c->regs, c->d & ByteOp);
836 c->modrm_val = *(unsigned long *)c->modrm_ptr;
840 if (c->ad_bytes == 2) {
841 unsigned bx = c->regs[VCPU_REGS_RBX];
842 unsigned bp = c->regs[VCPU_REGS_RBP];
843 unsigned si = c->regs[VCPU_REGS_RSI];
844 unsigned di = c->regs[VCPU_REGS_RDI];
846 /* 16-bit ModR/M decode. */
847 switch (c->modrm_mod) {
849 if (c->modrm_rm == 6)
850 c->modrm_ea += insn_fetch(u16, 2, c->eip);
853 c->modrm_ea += insn_fetch(s8, 1, c->eip);
856 c->modrm_ea += insn_fetch(u16, 2, c->eip);
859 switch (c->modrm_rm) {
861 c->modrm_ea += bx + si;
864 c->modrm_ea += bx + di;
867 c->modrm_ea += bp + si;
870 c->modrm_ea += bp + di;
879 if (c->modrm_mod != 0)
886 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
887 (c->modrm_rm == 6 && c->modrm_mod != 0))
888 if (!c->has_seg_override)
889 set_seg_override(c, VCPU_SREG_SS);
890 c->modrm_ea = (u16)c->modrm_ea;
892 /* 32/64-bit ModR/M decode. */
893 if ((c->modrm_rm & 7) == 4) {
894 sib = insn_fetch(u8, 1, c->eip);
895 index_reg |= (sib >> 3) & 7;
899 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
900 c->modrm_ea += insn_fetch(s32, 4, c->eip);
902 c->modrm_ea += c->regs[base_reg];
904 c->modrm_ea += c->regs[index_reg] << scale;
905 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
906 if (ctxt->mode == X86EMUL_MODE_PROT64)
909 c->modrm_ea += c->regs[c->modrm_rm];
910 switch (c->modrm_mod) {
912 if (c->modrm_rm == 5)
913 c->modrm_ea += insn_fetch(s32, 4, c->eip);
916 c->modrm_ea += insn_fetch(s8, 1, c->eip);
919 c->modrm_ea += insn_fetch(s32, 4, c->eip);
927 static int decode_abs(struct x86_emulate_ctxt *ctxt,
928 struct x86_emulate_ops *ops)
930 struct decode_cache *c = &ctxt->decode;
931 int rc = X86EMUL_CONTINUE;
933 switch (c->ad_bytes) {
935 c->modrm_ea = insn_fetch(u16, 2, c->eip);
938 c->modrm_ea = insn_fetch(u32, 4, c->eip);
941 c->modrm_ea = insn_fetch(u64, 8, c->eip);
949 x86_decode_insn(struct x86_emulate_ctxt *ctxt)
951 struct x86_emulate_ops *ops = ctxt->ops;
952 struct decode_cache *c = &ctxt->decode;
953 int rc = X86EMUL_CONTINUE;
954 int mode = ctxt->mode;
955 int def_op_bytes, def_ad_bytes, dual, goffset;
956 struct opcode opcode, *g_mod012, *g_mod3;
958 /* we cannot decode insn before we complete previous rep insn */
959 WARN_ON(ctxt->restart);
962 c->fetch.start = c->fetch.end = c->eip;
963 ctxt->cs_base = seg_base(ctxt, ops, VCPU_SREG_CS);
966 case X86EMUL_MODE_REAL:
967 case X86EMUL_MODE_VM86:
968 case X86EMUL_MODE_PROT16:
969 def_op_bytes = def_ad_bytes = 2;
971 case X86EMUL_MODE_PROT32:
972 def_op_bytes = def_ad_bytes = 4;
975 case X86EMUL_MODE_PROT64:
984 c->op_bytes = def_op_bytes;
985 c->ad_bytes = def_ad_bytes;
987 /* Legacy prefixes. */
989 switch (c->b = insn_fetch(u8, 1, c->eip)) {
990 case 0x66: /* operand-size override */
991 /* switch between 2/4 bytes */
992 c->op_bytes = def_op_bytes ^ 6;
994 case 0x67: /* address-size override */
995 if (mode == X86EMUL_MODE_PROT64)
996 /* switch between 4/8 bytes */
997 c->ad_bytes = def_ad_bytes ^ 12;
999 /* switch between 2/4 bytes */
1000 c->ad_bytes = def_ad_bytes ^ 6;
1002 case 0x26: /* ES override */
1003 case 0x2e: /* CS override */
1004 case 0x36: /* SS override */
1005 case 0x3e: /* DS override */
1006 set_seg_override(c, (c->b >> 3) & 3);
1008 case 0x64: /* FS override */
1009 case 0x65: /* GS override */
1010 set_seg_override(c, c->b & 7);
1012 case 0x40 ... 0x4f: /* REX */
1013 if (mode != X86EMUL_MODE_PROT64)
1015 c->rex_prefix = c->b;
1017 case 0xf0: /* LOCK */
1020 case 0xf2: /* REPNE/REPNZ */
1021 c->rep_prefix = REPNE_PREFIX;
1023 case 0xf3: /* REP/REPE/REPZ */
1024 c->rep_prefix = REPE_PREFIX;
1030 /* Any legacy prefix after a REX prefix nullifies its effect. */
1039 if (c->rex_prefix & 8)
1040 c->op_bytes = 8; /* REX.W */
1042 /* Opcode byte(s). */
1043 opcode = opcode_table[c->b];
1044 if (opcode.flags == 0) {
1045 /* Two-byte opcode? */
1048 c->b = insn_fetch(u8, 1, c->eip);
1049 opcode = twobyte_table[c->b];
1052 c->d = opcode.flags;
1055 dual = c->d & GroupDual;
1056 c->modrm = insn_fetch(u8, 1, c->eip);
1059 if (c->d & GroupDual) {
1060 g_mod012 = opcode.u.gdual->mod012;
1061 g_mod3 = opcode.u.gdual->mod3;
1063 g_mod012 = g_mod3 = opcode.u.group;
1065 c->d &= ~(Group | GroupDual);
1067 goffset = (c->modrm >> 3) & 7;
1069 if ((c->modrm >> 6) == 3)
1070 opcode = g_mod3[goffset];
1072 opcode = g_mod012[goffset];
1073 c->d |= opcode.flags;
1076 c->execute = opcode.u.execute;
1079 if (c->d == 0 || (c->d & Undefined)) {
1080 DPRINTF("Cannot emulate %02x\n", c->b);
1084 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
1087 /* ModRM and SIB bytes. */
1089 rc = decode_modrm(ctxt, ops);
1090 else if (c->d & MemAbs)
1091 rc = decode_abs(ctxt, ops);
1092 if (rc != X86EMUL_CONTINUE)
1095 if (!c->has_seg_override)
1096 set_seg_override(c, VCPU_SREG_DS);
1098 if (!(!c->twobyte && c->b == 0x8d))
1099 c->modrm_ea += seg_override_base(ctxt, ops, c);
1101 if (c->ad_bytes != 8)
1102 c->modrm_ea = (u32)c->modrm_ea;
1104 if (c->rip_relative)
1105 c->modrm_ea += c->eip;
1108 * Decode and fetch the source operand: register, memory
1111 switch (c->d & SrcMask) {
1115 decode_register_operand(&c->src, c, 0);
1124 c->src.bytes = (c->d & ByteOp) ? 1 :
1126 /* Don't fetch the address for invlpg: it could be unmapped. */
1127 if (c->twobyte && c->b == 0x01 && c->modrm_reg == 7)
1131 * For instructions with a ModR/M byte, switch to register
1132 * access if Mod = 3.
1134 if ((c->d & ModRM) && c->modrm_mod == 3) {
1135 c->src.type = OP_REG;
1136 c->src.val = c->modrm_val;
1137 c->src.ptr = c->modrm_ptr;
1140 c->src.type = OP_MEM;
1141 c->src.ptr = (unsigned long *)c->modrm_ea;
1146 c->src.type = OP_IMM;
1147 c->src.ptr = (unsigned long *)c->eip;
1148 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1149 if (c->src.bytes == 8)
1151 /* NB. Immediates are sign-extended as necessary. */
1152 switch (c->src.bytes) {
1154 c->src.val = insn_fetch(s8, 1, c->eip);
1157 c->src.val = insn_fetch(s16, 2, c->eip);
1160 c->src.val = insn_fetch(s32, 4, c->eip);
1163 if ((c->d & SrcMask) == SrcImmU) {
1164 switch (c->src.bytes) {
1169 c->src.val &= 0xffff;
1172 c->src.val &= 0xffffffff;
1179 c->src.type = OP_IMM;
1180 c->src.ptr = (unsigned long *)c->eip;
1182 if ((c->d & SrcMask) == SrcImmByte)
1183 c->src.val = insn_fetch(s8, 1, c->eip);
1185 c->src.val = insn_fetch(u8, 1, c->eip);
1188 c->src.type = OP_REG;
1189 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1190 c->src.ptr = &c->regs[VCPU_REGS_RAX];
1191 switch (c->src.bytes) {
1193 c->src.val = *(u8 *)c->src.ptr;
1196 c->src.val = *(u16 *)c->src.ptr;
1199 c->src.val = *(u32 *)c->src.ptr;
1202 c->src.val = *(u64 *)c->src.ptr;
1211 c->src.type = OP_MEM;
1212 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1213 c->src.ptr = (unsigned long *)
1214 register_address(c, seg_override_base(ctxt, ops, c),
1215 c->regs[VCPU_REGS_RSI]);
1219 c->src.type = OP_IMM;
1220 c->src.ptr = (unsigned long *)c->eip;
1221 c->src.bytes = c->op_bytes + 2;
1222 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
1225 c->src.type = OP_MEM;
1226 c->src.ptr = (unsigned long *)c->modrm_ea;
1227 c->src.bytes = c->op_bytes + 2;
1232 * Decode and fetch the second source operand: register, memory
1235 switch (c->d & Src2Mask) {
1240 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
1243 c->src2.type = OP_IMM;
1244 c->src2.ptr = (unsigned long *)c->eip;
1246 c->src2.val = insn_fetch(u8, 1, c->eip);
1254 /* Decode and fetch the destination operand: register or memory. */
1255 switch (c->d & DstMask) {
1257 /* Special instructions do their own operand decoding. */
1260 decode_register_operand(&c->dst, c,
1261 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
1265 if ((c->d & ModRM) && c->modrm_mod == 3) {
1266 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1267 c->dst.type = OP_REG;
1268 c->dst.val = c->dst.orig_val = c->modrm_val;
1269 c->dst.ptr = c->modrm_ptr;
1272 c->dst.type = OP_MEM;
1273 c->dst.ptr = (unsigned long *)c->modrm_ea;
1274 if ((c->d & DstMask) == DstMem64)
1277 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1280 unsigned long mask = ~(c->dst.bytes * 8 - 1);
1282 c->dst.ptr = (void *)c->dst.ptr +
1283 (c->src.val & mask) / 8;
1287 c->dst.type = OP_REG;
1288 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1289 c->dst.ptr = &c->regs[VCPU_REGS_RAX];
1290 switch (c->dst.bytes) {
1292 c->dst.val = *(u8 *)c->dst.ptr;
1295 c->dst.val = *(u16 *)c->dst.ptr;
1298 c->dst.val = *(u32 *)c->dst.ptr;
1301 c->dst.val = *(u64 *)c->dst.ptr;
1304 c->dst.orig_val = c->dst.val;
1307 c->dst.type = OP_MEM;
1308 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1309 c->dst.ptr = (unsigned long *)
1310 register_address(c, es_base(ctxt, ops),
1311 c->regs[VCPU_REGS_RDI]);
1317 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
1320 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1321 struct x86_emulate_ops *ops,
1322 unsigned long addr, void *dest, unsigned size)
1325 struct read_cache *mc = &ctxt->decode.mem_read;
1329 int n = min(size, 8u);
1331 if (mc->pos < mc->end)
1334 rc = ops->read_emulated(addr, mc->data + mc->end, n, &err,
1336 if (rc == X86EMUL_PROPAGATE_FAULT)
1337 emulate_pf(ctxt, addr, err);
1338 if (rc != X86EMUL_CONTINUE)
1343 memcpy(dest, mc->data + mc->pos, n);
1348 return X86EMUL_CONTINUE;
1351 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1352 struct x86_emulate_ops *ops,
1353 unsigned int size, unsigned short port,
1356 struct read_cache *rc = &ctxt->decode.io_read;
1358 if (rc->pos == rc->end) { /* refill pio read ahead */
1359 struct decode_cache *c = &ctxt->decode;
1360 unsigned int in_page, n;
1361 unsigned int count = c->rep_prefix ?
1362 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
1363 in_page = (ctxt->eflags & EFLG_DF) ?
1364 offset_in_page(c->regs[VCPU_REGS_RDI]) :
1365 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
1366 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1370 rc->pos = rc->end = 0;
1371 if (!ops->pio_in_emulated(size, port, rc->data, n, ctxt->vcpu))
1376 memcpy(dest, rc->data + rc->pos, size);
1381 static u32 desc_limit_scaled(struct desc_struct *desc)
1383 u32 limit = get_desc_limit(desc);
1385 return desc->g ? (limit << 12) | 0xfff : limit;
1388 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1389 struct x86_emulate_ops *ops,
1390 u16 selector, struct desc_ptr *dt)
1392 if (selector & 1 << 2) {
1393 struct desc_struct desc;
1394 memset (dt, 0, sizeof *dt);
1395 if (!ops->get_cached_descriptor(&desc, VCPU_SREG_LDTR, ctxt->vcpu))
1398 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1399 dt->address = get_desc_base(&desc);
1401 ops->get_gdt(dt, ctxt->vcpu);
1404 /* allowed just for 8 bytes segments */
1405 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1406 struct x86_emulate_ops *ops,
1407 u16 selector, struct desc_struct *desc)
1410 u16 index = selector >> 3;
1415 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1417 if (dt.size < index * 8 + 7) {
1418 emulate_gp(ctxt, selector & 0xfffc);
1419 return X86EMUL_PROPAGATE_FAULT;
1421 addr = dt.address + index * 8;
1422 ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu, &err);
1423 if (ret == X86EMUL_PROPAGATE_FAULT)
1424 emulate_pf(ctxt, addr, err);
1429 /* allowed just for 8 bytes segments */
1430 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1431 struct x86_emulate_ops *ops,
1432 u16 selector, struct desc_struct *desc)
1435 u16 index = selector >> 3;
1440 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1442 if (dt.size < index * 8 + 7) {
1443 emulate_gp(ctxt, selector & 0xfffc);
1444 return X86EMUL_PROPAGATE_FAULT;
1447 addr = dt.address + index * 8;
1448 ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu, &err);
1449 if (ret == X86EMUL_PROPAGATE_FAULT)
1450 emulate_pf(ctxt, addr, err);
1455 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1456 struct x86_emulate_ops *ops,
1457 u16 selector, int seg)
1459 struct desc_struct seg_desc;
1461 unsigned err_vec = GP_VECTOR;
1463 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1466 memset(&seg_desc, 0, sizeof seg_desc);
1468 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1469 || ctxt->mode == X86EMUL_MODE_REAL) {
1470 /* set real mode segment descriptor */
1471 set_desc_base(&seg_desc, selector << 4);
1472 set_desc_limit(&seg_desc, 0xffff);
1479 /* NULL selector is not valid for TR, CS and SS */
1480 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1484 /* TR should be in GDT only */
1485 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1488 if (null_selector) /* for NULL selector skip all following checks */
1491 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1492 if (ret != X86EMUL_CONTINUE)
1495 err_code = selector & 0xfffc;
1496 err_vec = GP_VECTOR;
1498 /* can't load system descriptor into segment selecor */
1499 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1503 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1509 cpl = ops->cpl(ctxt->vcpu);
1514 * segment is not a writable data segment or segment
1515 * selector's RPL != CPL or segment selector's RPL != CPL
1517 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1521 if (!(seg_desc.type & 8))
1524 if (seg_desc.type & 4) {
1530 if (rpl > cpl || dpl != cpl)
1533 /* CS(RPL) <- CPL */
1534 selector = (selector & 0xfffc) | cpl;
1537 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1540 case VCPU_SREG_LDTR:
1541 if (seg_desc.s || seg_desc.type != 2)
1544 default: /* DS, ES, FS, or GS */
1546 * segment is not a data or readable code segment or
1547 * ((segment is a data or nonconforming code segment)
1548 * and (both RPL and CPL > DPL))
1550 if ((seg_desc.type & 0xa) == 0x8 ||
1551 (((seg_desc.type & 0xc) != 0xc) &&
1552 (rpl > dpl && cpl > dpl)))
1558 /* mark segment as accessed */
1560 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1561 if (ret != X86EMUL_CONTINUE)
1565 ops->set_segment_selector(selector, seg, ctxt->vcpu);
1566 ops->set_cached_descriptor(&seg_desc, seg, ctxt->vcpu);
1567 return X86EMUL_CONTINUE;
1569 emulate_exception(ctxt, err_vec, err_code, true);
1570 return X86EMUL_PROPAGATE_FAULT;
1573 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1574 struct x86_emulate_ops *ops)
1577 struct decode_cache *c = &ctxt->decode;
1580 switch (c->dst.type) {
1582 /* The 4-byte case *is* correct:
1583 * in 64-bit mode we zero-extend.
1585 switch (c->dst.bytes) {
1587 *(u8 *)c->dst.ptr = (u8)c->dst.val;
1590 *(u16 *)c->dst.ptr = (u16)c->dst.val;
1593 *c->dst.ptr = (u32)c->dst.val;
1594 break; /* 64b: zero-ext */
1596 *c->dst.ptr = c->dst.val;
1602 rc = ops->cmpxchg_emulated(
1603 (unsigned long)c->dst.ptr,
1610 rc = ops->write_emulated(
1611 (unsigned long)c->dst.ptr,
1616 if (rc == X86EMUL_PROPAGATE_FAULT)
1618 (unsigned long)c->dst.ptr, err);
1619 if (rc != X86EMUL_CONTINUE)
1628 return X86EMUL_CONTINUE;
1631 static inline void emulate_push(struct x86_emulate_ctxt *ctxt,
1632 struct x86_emulate_ops *ops)
1634 struct decode_cache *c = &ctxt->decode;
1636 c->dst.type = OP_MEM;
1637 c->dst.bytes = c->op_bytes;
1638 c->dst.val = c->src.val;
1639 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1640 c->dst.ptr = (void *) register_address(c, ss_base(ctxt, ops),
1641 c->regs[VCPU_REGS_RSP]);
1644 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1645 struct x86_emulate_ops *ops,
1646 void *dest, int len)
1648 struct decode_cache *c = &ctxt->decode;
1651 rc = read_emulated(ctxt, ops, register_address(c, ss_base(ctxt, ops),
1652 c->regs[VCPU_REGS_RSP]),
1654 if (rc != X86EMUL_CONTINUE)
1657 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1661 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1662 struct x86_emulate_ops *ops,
1663 void *dest, int len)
1666 unsigned long val, change_mask;
1667 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1668 int cpl = ops->cpl(ctxt->vcpu);
1670 rc = emulate_pop(ctxt, ops, &val, len);
1671 if (rc != X86EMUL_CONTINUE)
1674 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1675 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1677 switch(ctxt->mode) {
1678 case X86EMUL_MODE_PROT64:
1679 case X86EMUL_MODE_PROT32:
1680 case X86EMUL_MODE_PROT16:
1682 change_mask |= EFLG_IOPL;
1684 change_mask |= EFLG_IF;
1686 case X86EMUL_MODE_VM86:
1688 emulate_gp(ctxt, 0);
1689 return X86EMUL_PROPAGATE_FAULT;
1691 change_mask |= EFLG_IF;
1693 default: /* real mode */
1694 change_mask |= (EFLG_IOPL | EFLG_IF);
1698 *(unsigned long *)dest =
1699 (ctxt->eflags & ~change_mask) | (val & change_mask);
1704 static void emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1705 struct x86_emulate_ops *ops, int seg)
1707 struct decode_cache *c = &ctxt->decode;
1709 c->src.val = ops->get_segment_selector(seg, ctxt->vcpu);
1711 emulate_push(ctxt, ops);
1714 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1715 struct x86_emulate_ops *ops, int seg)
1717 struct decode_cache *c = &ctxt->decode;
1718 unsigned long selector;
1721 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1722 if (rc != X86EMUL_CONTINUE)
1725 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1729 static int emulate_pusha(struct x86_emulate_ctxt *ctxt,
1730 struct x86_emulate_ops *ops)
1732 struct decode_cache *c = &ctxt->decode;
1733 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1734 int rc = X86EMUL_CONTINUE;
1735 int reg = VCPU_REGS_RAX;
1737 while (reg <= VCPU_REGS_RDI) {
1738 (reg == VCPU_REGS_RSP) ?
1739 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1741 emulate_push(ctxt, ops);
1743 rc = writeback(ctxt, ops);
1744 if (rc != X86EMUL_CONTINUE)
1750 /* Disable writeback. */
1751 c->dst.type = OP_NONE;
1756 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1757 struct x86_emulate_ops *ops)
1759 struct decode_cache *c = &ctxt->decode;
1760 int rc = X86EMUL_CONTINUE;
1761 int reg = VCPU_REGS_RDI;
1763 while (reg >= VCPU_REGS_RAX) {
1764 if (reg == VCPU_REGS_RSP) {
1765 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1770 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1771 if (rc != X86EMUL_CONTINUE)
1778 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
1779 struct x86_emulate_ops *ops)
1781 struct decode_cache *c = &ctxt->decode;
1782 int rc = X86EMUL_CONTINUE;
1783 unsigned long temp_eip = 0;
1784 unsigned long temp_eflags = 0;
1785 unsigned long cs = 0;
1786 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1787 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1788 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1789 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1791 /* TODO: Add stack limit check */
1793 rc = emulate_pop(ctxt, ops, &temp_eip, c->op_bytes);
1795 if (rc != X86EMUL_CONTINUE)
1798 if (temp_eip & ~0xffff) {
1799 emulate_gp(ctxt, 0);
1800 return X86EMUL_PROPAGATE_FAULT;
1803 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1805 if (rc != X86EMUL_CONTINUE)
1808 rc = emulate_pop(ctxt, ops, &temp_eflags, c->op_bytes);
1810 if (rc != X86EMUL_CONTINUE)
1813 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1815 if (rc != X86EMUL_CONTINUE)
1821 if (c->op_bytes == 4)
1822 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1823 else if (c->op_bytes == 2) {
1824 ctxt->eflags &= ~0xffff;
1825 ctxt->eflags |= temp_eflags;
1828 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1829 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1834 static inline int emulate_iret(struct x86_emulate_ctxt *ctxt,
1835 struct x86_emulate_ops* ops)
1837 switch(ctxt->mode) {
1838 case X86EMUL_MODE_REAL:
1839 return emulate_iret_real(ctxt, ops);
1840 case X86EMUL_MODE_VM86:
1841 case X86EMUL_MODE_PROT16:
1842 case X86EMUL_MODE_PROT32:
1843 case X86EMUL_MODE_PROT64:
1845 /* iret from protected mode unimplemented yet */
1846 return X86EMUL_UNHANDLEABLE;
1850 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1851 struct x86_emulate_ops *ops)
1853 struct decode_cache *c = &ctxt->decode;
1855 return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1858 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1860 struct decode_cache *c = &ctxt->decode;
1861 switch (c->modrm_reg) {
1863 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1866 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1869 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1872 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1874 case 4: /* sal/shl */
1875 case 6: /* sal/shl */
1876 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1879 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1882 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1887 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1888 struct x86_emulate_ops *ops)
1890 struct decode_cache *c = &ctxt->decode;
1892 switch (c->modrm_reg) {
1893 case 0 ... 1: /* test */
1894 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1897 c->dst.val = ~c->dst.val;
1900 emulate_1op("neg", c->dst, ctxt->eflags);
1908 static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
1909 struct x86_emulate_ops *ops)
1911 struct decode_cache *c = &ctxt->decode;
1913 switch (c->modrm_reg) {
1915 emulate_1op("inc", c->dst, ctxt->eflags);
1918 emulate_1op("dec", c->dst, ctxt->eflags);
1920 case 2: /* call near abs */ {
1923 c->eip = c->src.val;
1924 c->src.val = old_eip;
1925 emulate_push(ctxt, ops);
1928 case 4: /* jmp abs */
1929 c->eip = c->src.val;
1932 emulate_push(ctxt, ops);
1935 return X86EMUL_CONTINUE;
1938 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1939 struct x86_emulate_ops *ops)
1941 struct decode_cache *c = &ctxt->decode;
1942 u64 old = c->dst.orig_val64;
1944 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1945 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1946 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1947 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1948 ctxt->eflags &= ~EFLG_ZF;
1950 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1951 (u32) c->regs[VCPU_REGS_RBX];
1953 ctxt->eflags |= EFLG_ZF;
1955 return X86EMUL_CONTINUE;
1958 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1959 struct x86_emulate_ops *ops)
1961 struct decode_cache *c = &ctxt->decode;
1965 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1966 if (rc != X86EMUL_CONTINUE)
1968 if (c->op_bytes == 4)
1969 c->eip = (u32)c->eip;
1970 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1971 if (rc != X86EMUL_CONTINUE)
1973 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1978 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1979 struct x86_emulate_ops *ops, struct desc_struct *cs,
1980 struct desc_struct *ss)
1982 memset(cs, 0, sizeof(struct desc_struct));
1983 ops->get_cached_descriptor(cs, VCPU_SREG_CS, ctxt->vcpu);
1984 memset(ss, 0, sizeof(struct desc_struct));
1986 cs->l = 0; /* will be adjusted later */
1987 set_desc_base(cs, 0); /* flat segment */
1988 cs->g = 1; /* 4kb granularity */
1989 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1990 cs->type = 0x0b; /* Read, Execute, Accessed */
1992 cs->dpl = 0; /* will be adjusted later */
1996 set_desc_base(ss, 0); /* flat segment */
1997 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1998 ss->g = 1; /* 4kb granularity */
2000 ss->type = 0x03; /* Read/Write, Accessed */
2001 ss->d = 1; /* 32bit stack segment */
2007 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2009 struct decode_cache *c = &ctxt->decode;
2010 struct desc_struct cs, ss;
2014 /* syscall is not available in real mode */
2015 if (ctxt->mode == X86EMUL_MODE_REAL ||
2016 ctxt->mode == X86EMUL_MODE_VM86) {
2018 return X86EMUL_PROPAGATE_FAULT;
2021 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2023 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
2025 cs_sel = (u16)(msr_data & 0xfffc);
2026 ss_sel = (u16)(msr_data + 8);
2028 if (is_long_mode(ctxt->vcpu)) {
2032 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
2033 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
2034 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
2035 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
2037 c->regs[VCPU_REGS_RCX] = c->eip;
2038 if (is_long_mode(ctxt->vcpu)) {
2039 #ifdef CONFIG_X86_64
2040 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
2042 ops->get_msr(ctxt->vcpu,
2043 ctxt->mode == X86EMUL_MODE_PROT64 ?
2044 MSR_LSTAR : MSR_CSTAR, &msr_data);
2047 ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
2048 ctxt->eflags &= ~(msr_data | EFLG_RF);
2052 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
2053 c->eip = (u32)msr_data;
2055 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2058 return X86EMUL_CONTINUE;
2062 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2064 struct decode_cache *c = &ctxt->decode;
2065 struct desc_struct cs, ss;
2069 /* inject #GP if in real mode */
2070 if (ctxt->mode == X86EMUL_MODE_REAL) {
2071 emulate_gp(ctxt, 0);
2072 return X86EMUL_PROPAGATE_FAULT;
2075 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2076 * Therefore, we inject an #UD.
2078 if (ctxt->mode == X86EMUL_MODE_PROT64) {
2080 return X86EMUL_PROPAGATE_FAULT;
2083 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2085 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
2086 switch (ctxt->mode) {
2087 case X86EMUL_MODE_PROT32:
2088 if ((msr_data & 0xfffc) == 0x0) {
2089 emulate_gp(ctxt, 0);
2090 return X86EMUL_PROPAGATE_FAULT;
2093 case X86EMUL_MODE_PROT64:
2094 if (msr_data == 0x0) {
2095 emulate_gp(ctxt, 0);
2096 return X86EMUL_PROPAGATE_FAULT;
2101 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2102 cs_sel = (u16)msr_data;
2103 cs_sel &= ~SELECTOR_RPL_MASK;
2104 ss_sel = cs_sel + 8;
2105 ss_sel &= ~SELECTOR_RPL_MASK;
2106 if (ctxt->mode == X86EMUL_MODE_PROT64
2107 || is_long_mode(ctxt->vcpu)) {
2112 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
2113 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
2114 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
2115 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
2117 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
2120 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
2121 c->regs[VCPU_REGS_RSP] = msr_data;
2123 return X86EMUL_CONTINUE;
2127 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2129 struct decode_cache *c = &ctxt->decode;
2130 struct desc_struct cs, ss;
2135 /* inject #GP if in real mode or Virtual 8086 mode */
2136 if (ctxt->mode == X86EMUL_MODE_REAL ||
2137 ctxt->mode == X86EMUL_MODE_VM86) {
2138 emulate_gp(ctxt, 0);
2139 return X86EMUL_PROPAGATE_FAULT;
2142 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2144 if ((c->rex_prefix & 0x8) != 0x0)
2145 usermode = X86EMUL_MODE_PROT64;
2147 usermode = X86EMUL_MODE_PROT32;
2151 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
2153 case X86EMUL_MODE_PROT32:
2154 cs_sel = (u16)(msr_data + 16);
2155 if ((msr_data & 0xfffc) == 0x0) {
2156 emulate_gp(ctxt, 0);
2157 return X86EMUL_PROPAGATE_FAULT;
2159 ss_sel = (u16)(msr_data + 24);
2161 case X86EMUL_MODE_PROT64:
2162 cs_sel = (u16)(msr_data + 32);
2163 if (msr_data == 0x0) {
2164 emulate_gp(ctxt, 0);
2165 return X86EMUL_PROPAGATE_FAULT;
2167 ss_sel = cs_sel + 8;
2172 cs_sel |= SELECTOR_RPL_MASK;
2173 ss_sel |= SELECTOR_RPL_MASK;
2175 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
2176 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
2177 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
2178 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
2180 c->eip = c->regs[VCPU_REGS_RDX];
2181 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
2183 return X86EMUL_CONTINUE;
2186 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
2187 struct x86_emulate_ops *ops)
2190 if (ctxt->mode == X86EMUL_MODE_REAL)
2192 if (ctxt->mode == X86EMUL_MODE_VM86)
2194 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2195 return ops->cpl(ctxt->vcpu) > iopl;
2198 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2199 struct x86_emulate_ops *ops,
2202 struct desc_struct tr_seg;
2205 u8 perm, bit_idx = port & 0x7;
2206 unsigned mask = (1 << len) - 1;
2208 ops->get_cached_descriptor(&tr_seg, VCPU_SREG_TR, ctxt->vcpu);
2211 if (desc_limit_scaled(&tr_seg) < 103)
2213 r = ops->read_std(get_desc_base(&tr_seg) + 102, &io_bitmap_ptr, 2,
2215 if (r != X86EMUL_CONTINUE)
2217 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2219 r = ops->read_std(get_desc_base(&tr_seg) + io_bitmap_ptr + port/8,
2220 &perm, 1, ctxt->vcpu, NULL);
2221 if (r != X86EMUL_CONTINUE)
2223 if ((perm >> bit_idx) & mask)
2228 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2229 struct x86_emulate_ops *ops,
2232 if (emulator_bad_iopl(ctxt, ops))
2233 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
2238 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2239 struct x86_emulate_ops *ops,
2240 struct tss_segment_16 *tss)
2242 struct decode_cache *c = &ctxt->decode;
2245 tss->flag = ctxt->eflags;
2246 tss->ax = c->regs[VCPU_REGS_RAX];
2247 tss->cx = c->regs[VCPU_REGS_RCX];
2248 tss->dx = c->regs[VCPU_REGS_RDX];
2249 tss->bx = c->regs[VCPU_REGS_RBX];
2250 tss->sp = c->regs[VCPU_REGS_RSP];
2251 tss->bp = c->regs[VCPU_REGS_RBP];
2252 tss->si = c->regs[VCPU_REGS_RSI];
2253 tss->di = c->regs[VCPU_REGS_RDI];
2255 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2256 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2257 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2258 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2259 tss->ldt = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2262 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2263 struct x86_emulate_ops *ops,
2264 struct tss_segment_16 *tss)
2266 struct decode_cache *c = &ctxt->decode;
2270 ctxt->eflags = tss->flag | 2;
2271 c->regs[VCPU_REGS_RAX] = tss->ax;
2272 c->regs[VCPU_REGS_RCX] = tss->cx;
2273 c->regs[VCPU_REGS_RDX] = tss->dx;
2274 c->regs[VCPU_REGS_RBX] = tss->bx;
2275 c->regs[VCPU_REGS_RSP] = tss->sp;
2276 c->regs[VCPU_REGS_RBP] = tss->bp;
2277 c->regs[VCPU_REGS_RSI] = tss->si;
2278 c->regs[VCPU_REGS_RDI] = tss->di;
2281 * SDM says that segment selectors are loaded before segment
2284 ops->set_segment_selector(tss->ldt, VCPU_SREG_LDTR, ctxt->vcpu);
2285 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2286 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2287 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2288 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2291 * Now load segment descriptors. If fault happenes at this stage
2292 * it is handled in a context of new task
2294 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
2295 if (ret != X86EMUL_CONTINUE)
2297 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2298 if (ret != X86EMUL_CONTINUE)
2300 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2301 if (ret != X86EMUL_CONTINUE)
2303 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2304 if (ret != X86EMUL_CONTINUE)
2306 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2307 if (ret != X86EMUL_CONTINUE)
2310 return X86EMUL_CONTINUE;
2313 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2314 struct x86_emulate_ops *ops,
2315 u16 tss_selector, u16 old_tss_sel,
2316 ulong old_tss_base, struct desc_struct *new_desc)
2318 struct tss_segment_16 tss_seg;
2320 u32 err, new_tss_base = get_desc_base(new_desc);
2322 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2324 if (ret == X86EMUL_PROPAGATE_FAULT) {
2325 /* FIXME: need to provide precise fault address */
2326 emulate_pf(ctxt, old_tss_base, err);
2330 save_state_to_tss16(ctxt, ops, &tss_seg);
2332 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2334 if (ret == X86EMUL_PROPAGATE_FAULT) {
2335 /* FIXME: need to provide precise fault address */
2336 emulate_pf(ctxt, old_tss_base, err);
2340 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2342 if (ret == X86EMUL_PROPAGATE_FAULT) {
2343 /* FIXME: need to provide precise fault address */
2344 emulate_pf(ctxt, new_tss_base, err);
2348 if (old_tss_sel != 0xffff) {
2349 tss_seg.prev_task_link = old_tss_sel;
2351 ret = ops->write_std(new_tss_base,
2352 &tss_seg.prev_task_link,
2353 sizeof tss_seg.prev_task_link,
2355 if (ret == X86EMUL_PROPAGATE_FAULT) {
2356 /* FIXME: need to provide precise fault address */
2357 emulate_pf(ctxt, new_tss_base, err);
2362 return load_state_from_tss16(ctxt, ops, &tss_seg);
2365 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2366 struct x86_emulate_ops *ops,
2367 struct tss_segment_32 *tss)
2369 struct decode_cache *c = &ctxt->decode;
2371 tss->cr3 = ops->get_cr(3, ctxt->vcpu);
2373 tss->eflags = ctxt->eflags;
2374 tss->eax = c->regs[VCPU_REGS_RAX];
2375 tss->ecx = c->regs[VCPU_REGS_RCX];
2376 tss->edx = c->regs[VCPU_REGS_RDX];
2377 tss->ebx = c->regs[VCPU_REGS_RBX];
2378 tss->esp = c->regs[VCPU_REGS_RSP];
2379 tss->ebp = c->regs[VCPU_REGS_RBP];
2380 tss->esi = c->regs[VCPU_REGS_RSI];
2381 tss->edi = c->regs[VCPU_REGS_RDI];
2383 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2384 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2385 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2386 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2387 tss->fs = ops->get_segment_selector(VCPU_SREG_FS, ctxt->vcpu);
2388 tss->gs = ops->get_segment_selector(VCPU_SREG_GS, ctxt->vcpu);
2389 tss->ldt_selector = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2392 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2393 struct x86_emulate_ops *ops,
2394 struct tss_segment_32 *tss)
2396 struct decode_cache *c = &ctxt->decode;
2399 if (ops->set_cr(3, tss->cr3, ctxt->vcpu)) {
2400 emulate_gp(ctxt, 0);
2401 return X86EMUL_PROPAGATE_FAULT;
2404 ctxt->eflags = tss->eflags | 2;
2405 c->regs[VCPU_REGS_RAX] = tss->eax;
2406 c->regs[VCPU_REGS_RCX] = tss->ecx;
2407 c->regs[VCPU_REGS_RDX] = tss->edx;
2408 c->regs[VCPU_REGS_RBX] = tss->ebx;
2409 c->regs[VCPU_REGS_RSP] = tss->esp;
2410 c->regs[VCPU_REGS_RBP] = tss->ebp;
2411 c->regs[VCPU_REGS_RSI] = tss->esi;
2412 c->regs[VCPU_REGS_RDI] = tss->edi;
2415 * SDM says that segment selectors are loaded before segment
2418 ops->set_segment_selector(tss->ldt_selector, VCPU_SREG_LDTR, ctxt->vcpu);
2419 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2420 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2421 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2422 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2423 ops->set_segment_selector(tss->fs, VCPU_SREG_FS, ctxt->vcpu);
2424 ops->set_segment_selector(tss->gs, VCPU_SREG_GS, ctxt->vcpu);
2427 * Now load segment descriptors. If fault happenes at this stage
2428 * it is handled in a context of new task
2430 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2431 if (ret != X86EMUL_CONTINUE)
2433 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2434 if (ret != X86EMUL_CONTINUE)
2436 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2437 if (ret != X86EMUL_CONTINUE)
2439 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2440 if (ret != X86EMUL_CONTINUE)
2442 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2443 if (ret != X86EMUL_CONTINUE)
2445 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2446 if (ret != X86EMUL_CONTINUE)
2448 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2449 if (ret != X86EMUL_CONTINUE)
2452 return X86EMUL_CONTINUE;
2455 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2456 struct x86_emulate_ops *ops,
2457 u16 tss_selector, u16 old_tss_sel,
2458 ulong old_tss_base, struct desc_struct *new_desc)
2460 struct tss_segment_32 tss_seg;
2462 u32 err, new_tss_base = get_desc_base(new_desc);
2464 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2466 if (ret == X86EMUL_PROPAGATE_FAULT) {
2467 /* FIXME: need to provide precise fault address */
2468 emulate_pf(ctxt, old_tss_base, err);
2472 save_state_to_tss32(ctxt, ops, &tss_seg);
2474 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2476 if (ret == X86EMUL_PROPAGATE_FAULT) {
2477 /* FIXME: need to provide precise fault address */
2478 emulate_pf(ctxt, old_tss_base, err);
2482 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2484 if (ret == X86EMUL_PROPAGATE_FAULT) {
2485 /* FIXME: need to provide precise fault address */
2486 emulate_pf(ctxt, new_tss_base, err);
2490 if (old_tss_sel != 0xffff) {
2491 tss_seg.prev_task_link = old_tss_sel;
2493 ret = ops->write_std(new_tss_base,
2494 &tss_seg.prev_task_link,
2495 sizeof tss_seg.prev_task_link,
2497 if (ret == X86EMUL_PROPAGATE_FAULT) {
2498 /* FIXME: need to provide precise fault address */
2499 emulate_pf(ctxt, new_tss_base, err);
2504 return load_state_from_tss32(ctxt, ops, &tss_seg);
2507 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2508 struct x86_emulate_ops *ops,
2509 u16 tss_selector, int reason,
2510 bool has_error_code, u32 error_code)
2512 struct desc_struct curr_tss_desc, next_tss_desc;
2514 u16 old_tss_sel = ops->get_segment_selector(VCPU_SREG_TR, ctxt->vcpu);
2515 ulong old_tss_base =
2516 ops->get_cached_segment_base(VCPU_SREG_TR, ctxt->vcpu);
2519 /* FIXME: old_tss_base == ~0 ? */
2521 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2522 if (ret != X86EMUL_CONTINUE)
2524 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2525 if (ret != X86EMUL_CONTINUE)
2528 /* FIXME: check that next_tss_desc is tss */
2530 if (reason != TASK_SWITCH_IRET) {
2531 if ((tss_selector & 3) > next_tss_desc.dpl ||
2532 ops->cpl(ctxt->vcpu) > next_tss_desc.dpl) {
2533 emulate_gp(ctxt, 0);
2534 return X86EMUL_PROPAGATE_FAULT;
2538 desc_limit = desc_limit_scaled(&next_tss_desc);
2539 if (!next_tss_desc.p ||
2540 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2541 desc_limit < 0x2b)) {
2542 emulate_ts(ctxt, tss_selector & 0xfffc);
2543 return X86EMUL_PROPAGATE_FAULT;
2546 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2547 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2548 write_segment_descriptor(ctxt, ops, old_tss_sel,
2552 if (reason == TASK_SWITCH_IRET)
2553 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2555 /* set back link to prev task only if NT bit is set in eflags
2556 note that old_tss_sel is not used afetr this point */
2557 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2558 old_tss_sel = 0xffff;
2560 if (next_tss_desc.type & 8)
2561 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2562 old_tss_base, &next_tss_desc);
2564 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2565 old_tss_base, &next_tss_desc);
2566 if (ret != X86EMUL_CONTINUE)
2569 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2570 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2572 if (reason != TASK_SWITCH_IRET) {
2573 next_tss_desc.type |= (1 << 1); /* set busy flag */
2574 write_segment_descriptor(ctxt, ops, tss_selector,
2578 ops->set_cr(0, ops->get_cr(0, ctxt->vcpu) | X86_CR0_TS, ctxt->vcpu);
2579 ops->set_cached_descriptor(&next_tss_desc, VCPU_SREG_TR, ctxt->vcpu);
2580 ops->set_segment_selector(tss_selector, VCPU_SREG_TR, ctxt->vcpu);
2582 if (has_error_code) {
2583 struct decode_cache *c = &ctxt->decode;
2585 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2587 c->src.val = (unsigned long) error_code;
2588 emulate_push(ctxt, ops);
2594 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2595 u16 tss_selector, int reason,
2596 bool has_error_code, u32 error_code)
2598 struct x86_emulate_ops *ops = ctxt->ops;
2599 struct decode_cache *c = &ctxt->decode;
2603 c->dst.type = OP_NONE;
2605 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2606 has_error_code, error_code);
2608 if (rc == X86EMUL_CONTINUE) {
2609 rc = writeback(ctxt, ops);
2610 if (rc == X86EMUL_CONTINUE)
2614 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
2617 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned long base,
2618 int reg, struct operand *op)
2620 struct decode_cache *c = &ctxt->decode;
2621 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2623 register_address_increment(c, &c->regs[reg], df * op->bytes);
2624 op->ptr = (unsigned long *)register_address(c, base, c->regs[reg]);
2628 x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
2630 struct x86_emulate_ops *ops = ctxt->ops;
2632 struct decode_cache *c = &ctxt->decode;
2633 int rc = X86EMUL_CONTINUE;
2634 int saved_dst_type = c->dst.type;
2636 ctxt->decode.mem_read.pos = 0;
2638 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
2643 /* LOCK prefix is allowed only with some instructions */
2644 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
2649 /* Privileged instruction can be executed only in CPL=0 */
2650 if ((c->d & Priv) && ops->cpl(ctxt->vcpu)) {
2651 emulate_gp(ctxt, 0);
2655 if (c->rep_prefix && (c->d & String)) {
2656 ctxt->restart = true;
2657 /* All REP prefixes have the same first termination condition */
2658 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
2660 ctxt->restart = false;
2664 /* The second termination condition only applies for REPE
2665 * and REPNE. Test if the repeat string operation prefix is
2666 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
2667 * corresponding termination condition according to:
2668 * - if REPE/REPZ and ZF = 0 then done
2669 * - if REPNE/REPNZ and ZF = 1 then done
2671 if ((c->b == 0xa6) || (c->b == 0xa7) ||
2672 (c->b == 0xae) || (c->b == 0xaf)) {
2673 if ((c->rep_prefix == REPE_PREFIX) &&
2674 ((ctxt->eflags & EFLG_ZF) == 0))
2676 if ((c->rep_prefix == REPNE_PREFIX) &&
2677 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))
2683 if (c->src.type == OP_MEM) {
2684 rc = read_emulated(ctxt, ops, (unsigned long)c->src.ptr,
2685 c->src.valptr, c->src.bytes);
2686 if (rc != X86EMUL_CONTINUE)
2688 c->src.orig_val64 = c->src.val64;
2691 if (c->src2.type == OP_MEM) {
2692 rc = read_emulated(ctxt, ops, (unsigned long)c->src2.ptr,
2693 &c->src2.val, c->src2.bytes);
2694 if (rc != X86EMUL_CONTINUE)
2698 if ((c->d & DstMask) == ImplicitOps)
2702 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
2703 /* optimisation - avoid slow emulated read if Mov */
2704 rc = read_emulated(ctxt, ops, (unsigned long)c->dst.ptr,
2705 &c->dst.val, c->dst.bytes);
2706 if (rc != X86EMUL_CONTINUE)
2709 c->dst.orig_val = c->dst.val;
2714 rc = c->execute(ctxt);
2715 if (rc != X86EMUL_CONTINUE)
2726 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
2728 case 0x06: /* push es */
2729 emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
2731 case 0x07: /* pop es */
2732 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
2733 if (rc != X86EMUL_CONTINUE)
2738 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2740 case 0x0e: /* push cs */
2741 emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
2745 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
2747 case 0x16: /* push ss */
2748 emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
2750 case 0x17: /* pop ss */
2751 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
2752 if (rc != X86EMUL_CONTINUE)
2757 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
2759 case 0x1e: /* push ds */
2760 emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
2762 case 0x1f: /* pop ds */
2763 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
2764 if (rc != X86EMUL_CONTINUE)
2769 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
2773 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
2777 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
2781 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2783 case 0x40 ... 0x47: /* inc r16/r32 */
2784 emulate_1op("inc", c->dst, ctxt->eflags);
2786 case 0x48 ... 0x4f: /* dec r16/r32 */
2787 emulate_1op("dec", c->dst, ctxt->eflags);
2789 case 0x50 ... 0x57: /* push reg */
2790 emulate_push(ctxt, ops);
2792 case 0x58 ... 0x5f: /* pop reg */
2794 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
2795 if (rc != X86EMUL_CONTINUE)
2798 case 0x60: /* pusha */
2799 rc = emulate_pusha(ctxt, ops);
2800 if (rc != X86EMUL_CONTINUE)
2803 case 0x61: /* popa */
2804 rc = emulate_popa(ctxt, ops);
2805 if (rc != X86EMUL_CONTINUE)
2808 case 0x63: /* movsxd */
2809 if (ctxt->mode != X86EMUL_MODE_PROT64)
2810 goto cannot_emulate;
2811 c->dst.val = (s32) c->src.val;
2813 case 0x68: /* push imm */
2814 case 0x6a: /* push imm8 */
2815 emulate_push(ctxt, ops);
2817 case 0x6c: /* insb */
2818 case 0x6d: /* insw/insd */
2819 c->dst.bytes = min(c->dst.bytes, 4u);
2820 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2822 emulate_gp(ctxt, 0);
2825 if (!pio_in_emulated(ctxt, ops, c->dst.bytes,
2826 c->regs[VCPU_REGS_RDX], &c->dst.val))
2827 goto done; /* IO is needed, skip writeback */
2829 case 0x6e: /* outsb */
2830 case 0x6f: /* outsw/outsd */
2831 c->src.bytes = min(c->src.bytes, 4u);
2832 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2834 emulate_gp(ctxt, 0);
2837 ops->pio_out_emulated(c->src.bytes, c->regs[VCPU_REGS_RDX],
2838 &c->src.val, 1, ctxt->vcpu);
2840 c->dst.type = OP_NONE; /* nothing to writeback */
2842 case 0x70 ... 0x7f: /* jcc (short) */
2843 if (test_cc(c->b, ctxt->eflags))
2844 jmp_rel(c, c->src.val);
2846 case 0x80 ... 0x83: /* Grp1 */
2847 switch (c->modrm_reg) {
2868 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
2870 case 0x86 ... 0x87: /* xchg */
2872 /* Write back the register source. */
2873 switch (c->dst.bytes) {
2875 *(u8 *) c->src.ptr = (u8) c->dst.val;
2878 *(u16 *) c->src.ptr = (u16) c->dst.val;
2881 *c->src.ptr = (u32) c->dst.val;
2882 break; /* 64b reg: zero-extend */
2884 *c->src.ptr = c->dst.val;
2888 * Write back the memory destination with implicit LOCK
2891 c->dst.val = c->src.val;
2894 case 0x88 ... 0x8b: /* mov */
2896 case 0x8c: /* mov r/m, sreg */
2897 if (c->modrm_reg > VCPU_SREG_GS) {
2901 c->dst.val = ops->get_segment_selector(c->modrm_reg, ctxt->vcpu);
2903 case 0x8d: /* lea r16/r32, m */
2904 c->dst.val = c->modrm_ea;
2906 case 0x8e: { /* mov seg, r/m16 */
2911 if (c->modrm_reg == VCPU_SREG_CS ||
2912 c->modrm_reg > VCPU_SREG_GS) {
2917 if (c->modrm_reg == VCPU_SREG_SS)
2918 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2920 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
2922 c->dst.type = OP_NONE; /* Disable writeback. */
2925 case 0x8f: /* pop (sole member of Grp1a) */
2926 rc = emulate_grp1a(ctxt, ops);
2927 if (rc != X86EMUL_CONTINUE)
2930 case 0x90: /* nop / xchg r8,rax */
2931 if (c->dst.ptr == (unsigned long *)&c->regs[VCPU_REGS_RAX]) {
2932 c->dst.type = OP_NONE; /* nop */
2935 case 0x91 ... 0x97: /* xchg reg,rax */
2936 c->src.type = OP_REG;
2937 c->src.bytes = c->op_bytes;
2938 c->src.ptr = (unsigned long *) &c->regs[VCPU_REGS_RAX];
2939 c->src.val = *(c->src.ptr);
2941 case 0x9c: /* pushf */
2942 c->src.val = (unsigned long) ctxt->eflags;
2943 emulate_push(ctxt, ops);
2945 case 0x9d: /* popf */
2946 c->dst.type = OP_REG;
2947 c->dst.ptr = (unsigned long *) &ctxt->eflags;
2948 c->dst.bytes = c->op_bytes;
2949 rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
2950 if (rc != X86EMUL_CONTINUE)
2953 case 0xa0 ... 0xa3: /* mov */
2954 case 0xa4 ... 0xa5: /* movs */
2956 case 0xa6 ... 0xa7: /* cmps */
2957 c->dst.type = OP_NONE; /* Disable writeback. */
2958 DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.ptr, c->dst.ptr);
2960 case 0xa8 ... 0xa9: /* test ax, imm */
2962 case 0xaa ... 0xab: /* stos */
2963 c->dst.val = c->regs[VCPU_REGS_RAX];
2965 case 0xac ... 0xad: /* lods */
2967 case 0xae ... 0xaf: /* scas */
2968 DPRINTF("Urk! I don't handle SCAS.\n");
2969 goto cannot_emulate;
2970 case 0xb0 ... 0xbf: /* mov r, imm */
2975 case 0xc3: /* ret */
2976 c->dst.type = OP_REG;
2977 c->dst.ptr = &c->eip;
2978 c->dst.bytes = c->op_bytes;
2979 goto pop_instruction;
2980 case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */
2982 c->dst.val = c->src.val;
2984 case 0xcb: /* ret far */
2985 rc = emulate_ret_far(ctxt, ops);
2986 if (rc != X86EMUL_CONTINUE)
2989 case 0xcf: /* iret */
2990 rc = emulate_iret(ctxt, ops);
2992 if (rc != X86EMUL_CONTINUE)
2995 case 0xd0 ... 0xd1: /* Grp2 */
2999 case 0xd2 ... 0xd3: /* Grp2 */
3000 c->src.val = c->regs[VCPU_REGS_RCX];
3003 case 0xe4: /* inb */
3006 case 0xe6: /* outb */
3007 case 0xe7: /* out */
3009 case 0xe8: /* call (near) */ {
3010 long int rel = c->src.val;
3011 c->src.val = (unsigned long) c->eip;
3013 emulate_push(ctxt, ops);
3016 case 0xe9: /* jmp rel */
3018 case 0xea: { /* jmp far */
3021 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
3023 if (load_segment_descriptor(ctxt, ops, sel, VCPU_SREG_CS))
3027 memcpy(&c->eip, c->src.valptr, c->op_bytes);
3031 jmp: /* jmp rel short */
3032 jmp_rel(c, c->src.val);
3033 c->dst.type = OP_NONE; /* Disable writeback. */
3035 case 0xec: /* in al,dx */
3036 case 0xed: /* in (e/r)ax,dx */
3037 c->src.val = c->regs[VCPU_REGS_RDX];
3039 c->dst.bytes = min(c->dst.bytes, 4u);
3040 if (!emulator_io_permited(ctxt, ops, c->src.val, c->dst.bytes)) {
3041 emulate_gp(ctxt, 0);
3044 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
3046 goto done; /* IO is needed */
3048 case 0xee: /* out dx,al */
3049 case 0xef: /* out dx,(e/r)ax */
3050 c->src.val = c->regs[VCPU_REGS_RDX];
3052 c->dst.bytes = min(c->dst.bytes, 4u);
3053 if (!emulator_io_permited(ctxt, ops, c->src.val, c->dst.bytes)) {
3054 emulate_gp(ctxt, 0);
3057 ops->pio_out_emulated(c->dst.bytes, c->src.val, &c->dst.val, 1,
3059 c->dst.type = OP_NONE; /* Disable writeback. */
3061 case 0xf4: /* hlt */
3062 ctxt->vcpu->arch.halt_request = 1;
3064 case 0xf5: /* cmc */
3065 /* complement carry flag from eflags reg */
3066 ctxt->eflags ^= EFLG_CF;
3067 c->dst.type = OP_NONE; /* Disable writeback. */
3069 case 0xf6 ... 0xf7: /* Grp3 */
3070 if (!emulate_grp3(ctxt, ops))
3071 goto cannot_emulate;
3073 case 0xf8: /* clc */
3074 ctxt->eflags &= ~EFLG_CF;
3075 c->dst.type = OP_NONE; /* Disable writeback. */
3077 case 0xfa: /* cli */
3078 if (emulator_bad_iopl(ctxt, ops)) {
3079 emulate_gp(ctxt, 0);
3082 ctxt->eflags &= ~X86_EFLAGS_IF;
3083 c->dst.type = OP_NONE; /* Disable writeback. */
3086 case 0xfb: /* sti */
3087 if (emulator_bad_iopl(ctxt, ops)) {
3088 emulate_gp(ctxt, 0);
3091 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3092 ctxt->eflags |= X86_EFLAGS_IF;
3093 c->dst.type = OP_NONE; /* Disable writeback. */
3096 case 0xfc: /* cld */
3097 ctxt->eflags &= ~EFLG_DF;
3098 c->dst.type = OP_NONE; /* Disable writeback. */
3100 case 0xfd: /* std */
3101 ctxt->eflags |= EFLG_DF;
3102 c->dst.type = OP_NONE; /* Disable writeback. */
3104 case 0xfe: /* Grp4 */
3106 rc = emulate_grp45(ctxt, ops);
3107 if (rc != X86EMUL_CONTINUE)
3110 case 0xff: /* Grp5 */
3111 if (c->modrm_reg == 5)
3115 goto cannot_emulate;
3119 rc = writeback(ctxt, ops);
3120 if (rc != X86EMUL_CONTINUE)
3124 * restore dst type in case the decoding will be reused
3125 * (happens for string instruction )
3127 c->dst.type = saved_dst_type;
3129 if ((c->d & SrcMask) == SrcSI)
3130 string_addr_inc(ctxt, seg_override_base(ctxt, ops, c),
3131 VCPU_REGS_RSI, &c->src);
3133 if ((c->d & DstMask) == DstDI)
3134 string_addr_inc(ctxt, es_base(ctxt, ops), VCPU_REGS_RDI,
3137 if (c->rep_prefix && (c->d & String)) {
3138 struct read_cache *rc = &ctxt->decode.io_read;
3139 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3141 * Re-enter guest when pio read ahead buffer is empty or,
3142 * if it is not used, after each 1024 iteration.
3144 if ((rc->end == 0 && !(c->regs[VCPU_REGS_RCX] & 0x3ff)) ||
3145 (rc->end != 0 && rc->end == rc->pos))
3146 ctxt->restart = false;
3149 * reset read cache here in case string instruction is restared
3152 ctxt->decode.mem_read.end = 0;
3156 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
3160 case 0x01: /* lgdt, lidt, lmsw */
3161 switch (c->modrm_reg) {
3163 unsigned long address;
3165 case 0: /* vmcall */
3166 if (c->modrm_mod != 3 || c->modrm_rm != 1)
3167 goto cannot_emulate;
3169 rc = kvm_fix_hypercall(ctxt->vcpu);
3170 if (rc != X86EMUL_CONTINUE)
3173 /* Let the processor re-execute the fixed hypercall */
3175 /* Disable writeback. */
3176 c->dst.type = OP_NONE;
3179 rc = read_descriptor(ctxt, ops, c->src.ptr,
3180 &size, &address, c->op_bytes);
3181 if (rc != X86EMUL_CONTINUE)
3183 realmode_lgdt(ctxt->vcpu, size, address);
3184 /* Disable writeback. */
3185 c->dst.type = OP_NONE;
3187 case 3: /* lidt/vmmcall */
3188 if (c->modrm_mod == 3) {
3189 switch (c->modrm_rm) {
3191 rc = kvm_fix_hypercall(ctxt->vcpu);
3192 if (rc != X86EMUL_CONTINUE)
3196 goto cannot_emulate;
3199 rc = read_descriptor(ctxt, ops, c->src.ptr,
3202 if (rc != X86EMUL_CONTINUE)
3204 realmode_lidt(ctxt->vcpu, size, address);
3206 /* Disable writeback. */
3207 c->dst.type = OP_NONE;
3211 c->dst.val = ops->get_cr(0, ctxt->vcpu);
3214 ops->set_cr(0, (ops->get_cr(0, ctxt->vcpu) & ~0x0ful) |
3215 (c->src.val & 0x0f), ctxt->vcpu);
3216 c->dst.type = OP_NONE;
3218 case 5: /* not defined */
3222 emulate_invlpg(ctxt->vcpu, c->modrm_ea);
3223 /* Disable writeback. */
3224 c->dst.type = OP_NONE;
3227 goto cannot_emulate;
3230 case 0x05: /* syscall */
3231 rc = emulate_syscall(ctxt, ops);
3232 if (rc != X86EMUL_CONTINUE)
3238 emulate_clts(ctxt->vcpu);
3239 c->dst.type = OP_NONE;
3241 case 0x09: /* wbinvd */
3242 kvm_emulate_wbinvd(ctxt->vcpu);
3243 c->dst.type = OP_NONE;
3245 case 0x08: /* invd */
3246 case 0x0d: /* GrpP (prefetch) */
3247 case 0x18: /* Grp16 (prefetch/nop) */
3248 c->dst.type = OP_NONE;
3250 case 0x20: /* mov cr, reg */
3251 switch (c->modrm_reg) {
3258 c->regs[c->modrm_rm] = ops->get_cr(c->modrm_reg, ctxt->vcpu);
3259 c->dst.type = OP_NONE; /* no writeback */
3261 case 0x21: /* mov from dr to reg */
3262 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
3263 (c->modrm_reg == 4 || c->modrm_reg == 5)) {
3267 ops->get_dr(c->modrm_reg, &c->regs[c->modrm_rm], ctxt->vcpu);
3268 c->dst.type = OP_NONE; /* no writeback */
3270 case 0x22: /* mov reg, cr */
3271 if (ops->set_cr(c->modrm_reg, c->modrm_val, ctxt->vcpu)) {
3272 emulate_gp(ctxt, 0);
3275 c->dst.type = OP_NONE;
3277 case 0x23: /* mov from reg to dr */
3278 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
3279 (c->modrm_reg == 4 || c->modrm_reg == 5)) {
3284 if (ops->set_dr(c->modrm_reg, c->regs[c->modrm_rm] &
3285 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
3286 ~0ULL : ~0U), ctxt->vcpu) < 0) {
3287 /* #UD condition is already handled by the code above */
3288 emulate_gp(ctxt, 0);
3292 c->dst.type = OP_NONE; /* no writeback */
3296 msr_data = (u32)c->regs[VCPU_REGS_RAX]
3297 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
3298 if (ops->set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) {
3299 emulate_gp(ctxt, 0);
3302 rc = X86EMUL_CONTINUE;
3303 c->dst.type = OP_NONE;
3307 if (ops->get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) {
3308 emulate_gp(ctxt, 0);
3311 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
3312 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
3314 rc = X86EMUL_CONTINUE;
3315 c->dst.type = OP_NONE;
3317 case 0x34: /* sysenter */
3318 rc = emulate_sysenter(ctxt, ops);
3319 if (rc != X86EMUL_CONTINUE)
3324 case 0x35: /* sysexit */
3325 rc = emulate_sysexit(ctxt, ops);
3326 if (rc != X86EMUL_CONTINUE)
3331 case 0x40 ... 0x4f: /* cmov */
3332 c->dst.val = c->dst.orig_val = c->src.val;
3333 if (!test_cc(c->b, ctxt->eflags))
3334 c->dst.type = OP_NONE; /* no writeback */
3336 case 0x80 ... 0x8f: /* jnz rel, etc*/
3337 if (test_cc(c->b, ctxt->eflags))
3338 jmp_rel(c, c->src.val);
3339 c->dst.type = OP_NONE;
3341 case 0xa0: /* push fs */
3342 emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
3344 case 0xa1: /* pop fs */
3345 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
3346 if (rc != X86EMUL_CONTINUE)
3351 c->dst.type = OP_NONE;
3352 /* only subword offset */
3353 c->src.val &= (c->dst.bytes << 3) - 1;
3354 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
3356 case 0xa4: /* shld imm8, r, r/m */
3357 case 0xa5: /* shld cl, r, r/m */
3358 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
3360 case 0xa8: /* push gs */
3361 emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
3363 case 0xa9: /* pop gs */
3364 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
3365 if (rc != X86EMUL_CONTINUE)
3370 /* only subword offset */
3371 c->src.val &= (c->dst.bytes << 3) - 1;
3372 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
3374 case 0xac: /* shrd imm8, r, r/m */
3375 case 0xad: /* shrd cl, r, r/m */
3376 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
3378 case 0xae: /* clflush */
3380 case 0xb0 ... 0xb1: /* cmpxchg */
3382 * Save real source value, then compare EAX against
3385 c->src.orig_val = c->src.val;
3386 c->src.val = c->regs[VCPU_REGS_RAX];
3387 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
3388 if (ctxt->eflags & EFLG_ZF) {
3389 /* Success: write back to memory. */
3390 c->dst.val = c->src.orig_val;
3392 /* Failure: write the value we saw to EAX. */
3393 c->dst.type = OP_REG;
3394 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
3399 /* only subword offset */
3400 c->src.val &= (c->dst.bytes << 3) - 1;
3401 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
3403 case 0xb6 ... 0xb7: /* movzx */
3404 c->dst.bytes = c->op_bytes;
3405 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
3408 case 0xba: /* Grp8 */
3409 switch (c->modrm_reg & 3) {
3422 /* only subword offset */
3423 c->src.val &= (c->dst.bytes << 3) - 1;
3424 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
3426 case 0xbe ... 0xbf: /* movsx */
3427 c->dst.bytes = c->op_bytes;
3428 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
3431 case 0xc3: /* movnti */
3432 c->dst.bytes = c->op_bytes;
3433 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
3436 case 0xc7: /* Grp9 (cmpxchg8b) */
3437 rc = emulate_grp9(ctxt, ops);
3438 if (rc != X86EMUL_CONTINUE)
3442 goto cannot_emulate;
3447 DPRINTF("Cannot emulate %02x\n", c->b);