1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
13 * Avi Kivity <avi@qumranet.com>
14 * Yaniv Kamay <yaniv@qumranet.com>
16 * This work is licensed under the terms of the GNU GPL, version 2. See
17 * the COPYING file in the top-level directory.
19 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
25 #include <public/xen.h>
26 #define DPRINTF(_f, _a ...) printf(_f , ## _a)
28 #include <linux/kvm_host.h>
29 #include "kvm_cache_regs.h"
30 #define DPRINTF(x...) do {} while (0)
32 #include <linux/module.h>
33 #include <asm/kvm_emulate.h>
38 * Opcode effective-address decode tables.
39 * Note that we only emulate instructions that have at least one memory
40 * operand (excluding implicit stack references). We assume that stack
41 * references and instruction fetches will never occur in special memory
42 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
46 /* Operand sizes: 8-bit operands or specified/overridden size. */
47 #define ByteOp (1<<0) /* 8-bit operands. */
48 /* Destination operand type. */
49 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
50 #define DstReg (2<<1) /* Register operand. */
51 #define DstMem (3<<1) /* Memory operand. */
52 #define DstAcc (4<<1) /* Destination Accumulator */
53 #define DstMask (7<<1)
54 /* Source operand type. */
55 #define SrcNone (0<<4) /* No source operand. */
56 #define SrcImplicit (0<<4) /* Source operand is implicit in the opcode. */
57 #define SrcReg (1<<4) /* Register operand. */
58 #define SrcMem (2<<4) /* Memory operand. */
59 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
60 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
61 #define SrcImm (5<<4) /* Immediate operand. */
62 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
63 #define SrcOne (7<<4) /* Implied '1' */
64 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
65 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
66 #define SrcMask (0xf<<4)
67 /* Generic ModRM decode. */
69 /* Destination is only written; never read. */
72 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
73 #define String (1<<12) /* String instruction (rep capable) */
74 #define Stack (1<<13) /* Stack instruction (push/pop) */
75 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
76 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
77 #define GroupMask 0xff /* Group number stored in bits 0:7 */
80 /* Source 2 operand type */
81 #define Src2None (0<<29)
82 #define Src2CL (1<<29)
83 #define Src2ImmByte (2<<29)
84 #define Src2One (3<<29)
85 #define Src2Imm16 (4<<29)
86 #define Src2Mask (7<<29)
89 Group1_80, Group1_81, Group1_82, Group1_83,
90 Group1A, Group3_Byte, Group3, Group4, Group5, Group7,
94 static u32 opcode_table[256] = {
96 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
97 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
98 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
99 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
101 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
102 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
103 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
104 ImplicitOps | Stack | No64, 0,
106 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
107 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
108 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
109 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
111 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
112 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
113 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
114 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
116 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
117 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
118 DstAcc | SrcImmByte, DstAcc | SrcImm, 0, 0,
120 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
121 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
124 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
125 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
128 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
129 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
130 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
133 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
135 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
137 SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack,
138 SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack,
140 DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack,
141 DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack,
143 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
144 0, DstReg | SrcMem32 | ModRM | Mov /* movsxd (x86/64) */ ,
147 SrcImm | Mov | Stack, 0, SrcImmByte | Mov | Stack, 0,
148 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* insb, insw/insd */
149 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* outsb, outsw/outsd */
151 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
152 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
154 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
155 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
157 Group | Group1_80, Group | Group1_81,
158 Group | Group1_82, Group | Group1_83,
159 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
160 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
162 ByteOp | DstMem | SrcReg | ModRM | Mov, DstMem | SrcReg | ModRM | Mov,
163 ByteOp | DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
164 DstMem | SrcReg | ModRM | Mov, ModRM | DstReg,
165 DstReg | SrcMem | ModRM | Mov, Group | Group1A,
167 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
169 0, 0, SrcImm | Src2Imm16 | No64, 0,
170 ImplicitOps | Stack, ImplicitOps | Stack, 0, 0,
172 ByteOp | DstReg | SrcMem | Mov | MemAbs, DstReg | SrcMem | Mov | MemAbs,
173 ByteOp | DstMem | SrcReg | Mov | MemAbs, DstMem | SrcReg | Mov | MemAbs,
174 ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
175 ByteOp | ImplicitOps | String, ImplicitOps | String,
177 0, 0, ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
178 ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
179 ByteOp | ImplicitOps | String, ImplicitOps | String,
181 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
182 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
183 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
184 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
186 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
187 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
188 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
189 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
191 ByteOp | DstMem | SrcImm | ModRM, DstMem | SrcImmByte | ModRM,
192 0, ImplicitOps | Stack, 0, 0,
193 ByteOp | DstMem | SrcImm | ModRM | Mov, DstMem | SrcImm | ModRM | Mov,
195 0, 0, 0, ImplicitOps | Stack,
196 ImplicitOps, SrcImmByte, ImplicitOps | No64, ImplicitOps,
198 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
199 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
202 0, 0, 0, 0, 0, 0, 0, 0,
205 ByteOp | SrcImmUByte, SrcImmUByte,
206 ByteOp | SrcImmUByte, SrcImmUByte,
208 SrcImm | Stack, SrcImm | ImplicitOps,
209 SrcImmU | Src2Imm16 | No64, SrcImmByte | ImplicitOps,
210 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps,
211 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps,
214 ImplicitOps, ImplicitOps, Group | Group3_Byte, Group | Group3,
216 ImplicitOps, 0, ImplicitOps, ImplicitOps,
217 ImplicitOps, ImplicitOps, Group | Group4, Group | Group5,
220 static u32 twobyte_table[256] = {
222 0, Group | GroupDual | Group7, 0, 0, 0, ImplicitOps, ImplicitOps, 0,
223 ImplicitOps, ImplicitOps, 0, 0, 0, ImplicitOps | ModRM, 0, 0,
225 0, 0, 0, 0, 0, 0, 0, 0, ImplicitOps | ModRM, 0, 0, 0, 0, 0, 0, 0,
227 ModRM | ImplicitOps, ModRM, ModRM | ImplicitOps, ModRM, 0, 0, 0, 0,
228 0, 0, 0, 0, 0, 0, 0, 0,
230 ImplicitOps, 0, ImplicitOps, 0,
231 ImplicitOps, ImplicitOps, 0, 0,
232 0, 0, 0, 0, 0, 0, 0, 0,
234 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
235 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
236 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
237 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
239 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
240 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
241 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
242 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
244 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
246 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
248 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
250 SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm,
251 SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm,
253 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
255 ImplicitOps | Stack, ImplicitOps | Stack,
256 0, DstMem | SrcReg | ModRM | BitOp,
257 DstMem | SrcReg | Src2ImmByte | ModRM,
258 DstMem | SrcReg | Src2CL | ModRM, 0, 0,
260 ImplicitOps | Stack, ImplicitOps | Stack,
261 0, DstMem | SrcReg | ModRM | BitOp,
262 DstMem | SrcReg | Src2ImmByte | ModRM,
263 DstMem | SrcReg | Src2CL | ModRM,
266 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, 0,
267 DstMem | SrcReg | ModRM | BitOp,
268 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
269 DstReg | SrcMem16 | ModRM | Mov,
271 0, 0, Group | Group8, DstMem | SrcReg | ModRM | BitOp,
272 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
273 DstReg | SrcMem16 | ModRM | Mov,
275 0, 0, 0, DstMem | SrcReg | ModRM | Mov,
276 0, 0, 0, Group | GroupDual | Group9,
277 0, 0, 0, 0, 0, 0, 0, 0,
279 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
281 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
283 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
286 static u32 group_table[] = {
288 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
289 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
290 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
291 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
293 DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
294 DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
295 DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
296 DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
298 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
299 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
300 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
301 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
303 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
304 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
305 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
306 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
308 DstMem | SrcNone | ModRM | Mov | Stack, 0, 0, 0, 0, 0, 0, 0,
310 ByteOp | SrcImm | DstMem | ModRM, 0,
311 ByteOp | DstMem | SrcNone | ModRM, ByteOp | DstMem | SrcNone | ModRM,
314 DstMem | SrcImm | ModRM, 0,
315 DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM,
318 ByteOp | DstMem | SrcNone | ModRM, ByteOp | DstMem | SrcNone | ModRM,
321 DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM,
322 SrcMem | ModRM | Stack, 0,
323 SrcMem | ModRM | Stack, 0, SrcMem | ModRM | Stack, 0,
325 0, 0, ModRM | SrcMem, ModRM | SrcMem,
326 SrcNone | ModRM | DstMem | Mov, 0,
327 SrcMem16 | ModRM | Mov, SrcMem | ModRM | ByteOp,
330 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
331 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
333 0, ImplicitOps | ModRM, 0, 0, 0, 0, 0, 0,
336 static u32 group2_table[] = {
338 SrcNone | ModRM, 0, 0, SrcNone | ModRM,
339 SrcNone | ModRM | DstMem | Mov, 0,
340 SrcMem16 | ModRM | Mov, 0,
342 0, 0, 0, 0, 0, 0, 0, 0,
345 /* EFLAGS bit definitions. */
346 #define EFLG_VM (1<<17)
347 #define EFLG_RF (1<<16)
348 #define EFLG_OF (1<<11)
349 #define EFLG_DF (1<<10)
350 #define EFLG_IF (1<<9)
351 #define EFLG_SF (1<<7)
352 #define EFLG_ZF (1<<6)
353 #define EFLG_AF (1<<4)
354 #define EFLG_PF (1<<2)
355 #define EFLG_CF (1<<0)
358 * Instruction emulation:
359 * Most instructions are emulated directly via a fragment of inline assembly
360 * code. This allows us to save/restore EFLAGS and thus very easily pick up
361 * any modified flags.
364 #if defined(CONFIG_X86_64)
365 #define _LO32 "k" /* force 32-bit operand */
366 #define _STK "%%rsp" /* stack pointer */
367 #elif defined(__i386__)
368 #define _LO32 "" /* force 32-bit operand */
369 #define _STK "%%esp" /* stack pointer */
373 * These EFLAGS bits are restored from saved value during emulation, and
374 * any changes are written back to the saved value after emulation.
376 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
378 /* Before executing instruction: restore necessary bits in EFLAGS. */
379 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
380 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
381 "movl %"_sav",%"_LO32 _tmp"; " \
384 "movl %"_msk",%"_LO32 _tmp"; " \
385 "andl %"_LO32 _tmp",("_STK"); " \
387 "notl %"_LO32 _tmp"; " \
388 "andl %"_LO32 _tmp",("_STK"); " \
389 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
391 "orl %"_LO32 _tmp",("_STK"); " \
395 /* After executing instruction: write-back necessary bits in EFLAGS. */
396 #define _POST_EFLAGS(_sav, _msk, _tmp) \
397 /* _sav |= EFLAGS & _msk; */ \
400 "andl %"_msk",%"_LO32 _tmp"; " \
401 "orl %"_LO32 _tmp",%"_sav"; "
409 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix) \
411 __asm__ __volatile__ ( \
412 _PRE_EFLAGS("0", "4", "2") \
413 _op _suffix " %"_x"3,%1; " \
414 _POST_EFLAGS("0", "4", "2") \
415 : "=m" (_eflags), "=m" ((_dst).val), \
417 : _y ((_src).val), "i" (EFLAGS_MASK)); \
421 /* Raw emulation: instruction has two explicit operands. */
422 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
424 unsigned long _tmp; \
426 switch ((_dst).bytes) { \
428 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w"); \
431 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l"); \
434 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q")); \
439 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
441 unsigned long _tmp; \
442 switch ((_dst).bytes) { \
444 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b"); \
447 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
448 _wx, _wy, _lx, _ly, _qx, _qy); \
453 /* Source operand is byte-sized and may be restricted to just %cl. */
454 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
455 __emulate_2op(_op, _src, _dst, _eflags, \
456 "b", "c", "b", "c", "b", "c", "b", "c")
458 /* Source operand is byte, word, long or quad sized. */
459 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
460 __emulate_2op(_op, _src, _dst, _eflags, \
461 "b", "q", "w", "r", _LO32, "r", "", "r")
463 /* Source operand is word, long or quad sized. */
464 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
465 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
466 "w", "r", _LO32, "r", "", "r")
468 /* Instruction has three operands and one operand is stored in ECX register */
469 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
471 unsigned long _tmp; \
472 _type _clv = (_cl).val; \
473 _type _srcv = (_src).val; \
474 _type _dstv = (_dst).val; \
476 __asm__ __volatile__ ( \
477 _PRE_EFLAGS("0", "5", "2") \
478 _op _suffix " %4,%1 \n" \
479 _POST_EFLAGS("0", "5", "2") \
480 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
481 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
484 (_cl).val = (unsigned long) _clv; \
485 (_src).val = (unsigned long) _srcv; \
486 (_dst).val = (unsigned long) _dstv; \
489 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
491 switch ((_dst).bytes) { \
493 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
494 "w", unsigned short); \
497 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
498 "l", unsigned int); \
501 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
502 "q", unsigned long)); \
507 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
509 unsigned long _tmp; \
511 __asm__ __volatile__ ( \
512 _PRE_EFLAGS("0", "3", "2") \
513 _op _suffix " %1; " \
514 _POST_EFLAGS("0", "3", "2") \
515 : "=m" (_eflags), "+m" ((_dst).val), \
517 : "i" (EFLAGS_MASK)); \
520 /* Instruction has only one explicit operand (no source operand). */
521 #define emulate_1op(_op, _dst, _eflags) \
523 switch ((_dst).bytes) { \
524 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
525 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
526 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
527 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
531 /* Fetch next part of the instruction being emulated. */
532 #define insn_fetch(_type, _size, _eip) \
533 ({ unsigned long _x; \
534 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
541 static inline unsigned long ad_mask(struct decode_cache *c)
543 return (1UL << (c->ad_bytes << 3)) - 1;
546 /* Access/update address held in a register, based on addressing mode. */
547 static inline unsigned long
548 address_mask(struct decode_cache *c, unsigned long reg)
550 if (c->ad_bytes == sizeof(unsigned long))
553 return reg & ad_mask(c);
556 static inline unsigned long
557 register_address(struct decode_cache *c, unsigned long base, unsigned long reg)
559 return base + address_mask(c, reg);
563 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
565 if (c->ad_bytes == sizeof(unsigned long))
568 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
571 static inline void jmp_rel(struct decode_cache *c, int rel)
573 register_address_increment(c, &c->eip, rel);
576 static void set_seg_override(struct decode_cache *c, int seg)
578 c->has_seg_override = true;
579 c->seg_override = seg;
582 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
584 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
587 return kvm_x86_ops->get_segment_base(ctxt->vcpu, seg);
590 static unsigned long seg_override_base(struct x86_emulate_ctxt *ctxt,
591 struct decode_cache *c)
593 if (!c->has_seg_override)
596 return seg_base(ctxt, c->seg_override);
599 static unsigned long es_base(struct x86_emulate_ctxt *ctxt)
601 return seg_base(ctxt, VCPU_SREG_ES);
604 static unsigned long ss_base(struct x86_emulate_ctxt *ctxt)
606 return seg_base(ctxt, VCPU_SREG_SS);
609 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
610 struct x86_emulate_ops *ops,
611 unsigned long linear, u8 *dest)
613 struct fetch_cache *fc = &ctxt->decode.fetch;
617 if (linear < fc->start || linear >= fc->end) {
618 size = min(15UL, PAGE_SIZE - offset_in_page(linear));
619 rc = ops->read_std(linear, fc->data, size, ctxt->vcpu);
623 fc->end = linear + size;
625 *dest = fc->data[linear - fc->start];
629 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
630 struct x86_emulate_ops *ops,
631 unsigned long eip, void *dest, unsigned size)
635 /* x86 instructions are limited to 15 bytes. */
636 if (eip + size - ctxt->decode.eip_orig > 15)
637 return X86EMUL_UNHANDLEABLE;
638 eip += ctxt->cs_base;
640 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
648 * Given the 'reg' portion of a ModRM byte, and a register block, return a
649 * pointer into the block that addresses the relevant register.
650 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
652 static void *decode_register(u8 modrm_reg, unsigned long *regs,
657 p = ®s[modrm_reg];
658 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
659 p = (unsigned char *)®s[modrm_reg & 3] + 1;
663 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
664 struct x86_emulate_ops *ops,
666 u16 *size, unsigned long *address, int op_bytes)
673 rc = ops->read_std((unsigned long)ptr, (unsigned long *)size, 2,
677 rc = ops->read_std((unsigned long)ptr + 2, address, op_bytes,
682 static int test_cc(unsigned int condition, unsigned int flags)
686 switch ((condition & 15) >> 1) {
688 rc |= (flags & EFLG_OF);
690 case 1: /* b/c/nae */
691 rc |= (flags & EFLG_CF);
694 rc |= (flags & EFLG_ZF);
697 rc |= (flags & (EFLG_CF|EFLG_ZF));
700 rc |= (flags & EFLG_SF);
703 rc |= (flags & EFLG_PF);
706 rc |= (flags & EFLG_ZF);
709 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
713 /* Odd condition identifiers (lsb == 1) have inverted sense. */
714 return (!!rc ^ (condition & 1));
717 static void decode_register_operand(struct operand *op,
718 struct decode_cache *c,
721 unsigned reg = c->modrm_reg;
722 int highbyte_regs = c->rex_prefix == 0;
725 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
727 if ((c->d & ByteOp) && !inhibit_bytereg) {
728 op->ptr = decode_register(reg, c->regs, highbyte_regs);
729 op->val = *(u8 *)op->ptr;
732 op->ptr = decode_register(reg, c->regs, 0);
733 op->bytes = c->op_bytes;
736 op->val = *(u16 *)op->ptr;
739 op->val = *(u32 *)op->ptr;
742 op->val = *(u64 *) op->ptr;
746 op->orig_val = op->val;
749 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
750 struct x86_emulate_ops *ops)
752 struct decode_cache *c = &ctxt->decode;
754 int index_reg = 0, base_reg = 0, scale;
758 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
759 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
760 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
763 c->modrm = insn_fetch(u8, 1, c->eip);
764 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
765 c->modrm_reg |= (c->modrm & 0x38) >> 3;
766 c->modrm_rm |= (c->modrm & 0x07);
770 if (c->modrm_mod == 3) {
771 c->modrm_ptr = decode_register(c->modrm_rm,
772 c->regs, c->d & ByteOp);
773 c->modrm_val = *(unsigned long *)c->modrm_ptr;
777 if (c->ad_bytes == 2) {
778 unsigned bx = c->regs[VCPU_REGS_RBX];
779 unsigned bp = c->regs[VCPU_REGS_RBP];
780 unsigned si = c->regs[VCPU_REGS_RSI];
781 unsigned di = c->regs[VCPU_REGS_RDI];
783 /* 16-bit ModR/M decode. */
784 switch (c->modrm_mod) {
786 if (c->modrm_rm == 6)
787 c->modrm_ea += insn_fetch(u16, 2, c->eip);
790 c->modrm_ea += insn_fetch(s8, 1, c->eip);
793 c->modrm_ea += insn_fetch(u16, 2, c->eip);
796 switch (c->modrm_rm) {
798 c->modrm_ea += bx + si;
801 c->modrm_ea += bx + di;
804 c->modrm_ea += bp + si;
807 c->modrm_ea += bp + di;
816 if (c->modrm_mod != 0)
823 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
824 (c->modrm_rm == 6 && c->modrm_mod != 0))
825 if (!c->has_seg_override)
826 set_seg_override(c, VCPU_SREG_SS);
827 c->modrm_ea = (u16)c->modrm_ea;
829 /* 32/64-bit ModR/M decode. */
830 if ((c->modrm_rm & 7) == 4) {
831 sib = insn_fetch(u8, 1, c->eip);
832 index_reg |= (sib >> 3) & 7;
836 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
837 c->modrm_ea += insn_fetch(s32, 4, c->eip);
839 c->modrm_ea += c->regs[base_reg];
841 c->modrm_ea += c->regs[index_reg] << scale;
842 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
843 if (ctxt->mode == X86EMUL_MODE_PROT64)
846 c->modrm_ea += c->regs[c->modrm_rm];
847 switch (c->modrm_mod) {
849 if (c->modrm_rm == 5)
850 c->modrm_ea += insn_fetch(s32, 4, c->eip);
853 c->modrm_ea += insn_fetch(s8, 1, c->eip);
856 c->modrm_ea += insn_fetch(s32, 4, c->eip);
864 static int decode_abs(struct x86_emulate_ctxt *ctxt,
865 struct x86_emulate_ops *ops)
867 struct decode_cache *c = &ctxt->decode;
870 switch (c->ad_bytes) {
872 c->modrm_ea = insn_fetch(u16, 2, c->eip);
875 c->modrm_ea = insn_fetch(u32, 4, c->eip);
878 c->modrm_ea = insn_fetch(u64, 8, c->eip);
886 x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
888 struct decode_cache *c = &ctxt->decode;
890 int mode = ctxt->mode;
891 int def_op_bytes, def_ad_bytes, group;
893 /* Shadow copy of register state. Committed on successful emulation. */
895 memset(c, 0, sizeof(struct decode_cache));
896 c->eip = c->eip_orig = kvm_rip_read(ctxt->vcpu);
897 ctxt->cs_base = seg_base(ctxt, VCPU_SREG_CS);
898 memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs);
901 case X86EMUL_MODE_REAL:
902 case X86EMUL_MODE_PROT16:
903 def_op_bytes = def_ad_bytes = 2;
905 case X86EMUL_MODE_PROT32:
906 def_op_bytes = def_ad_bytes = 4;
909 case X86EMUL_MODE_PROT64:
918 c->op_bytes = def_op_bytes;
919 c->ad_bytes = def_ad_bytes;
921 /* Legacy prefixes. */
923 switch (c->b = insn_fetch(u8, 1, c->eip)) {
924 case 0x66: /* operand-size override */
925 /* switch between 2/4 bytes */
926 c->op_bytes = def_op_bytes ^ 6;
928 case 0x67: /* address-size override */
929 if (mode == X86EMUL_MODE_PROT64)
930 /* switch between 4/8 bytes */
931 c->ad_bytes = def_ad_bytes ^ 12;
933 /* switch between 2/4 bytes */
934 c->ad_bytes = def_ad_bytes ^ 6;
936 case 0x26: /* ES override */
937 case 0x2e: /* CS override */
938 case 0x36: /* SS override */
939 case 0x3e: /* DS override */
940 set_seg_override(c, (c->b >> 3) & 3);
942 case 0x64: /* FS override */
943 case 0x65: /* GS override */
944 set_seg_override(c, c->b & 7);
946 case 0x40 ... 0x4f: /* REX */
947 if (mode != X86EMUL_MODE_PROT64)
949 c->rex_prefix = c->b;
951 case 0xf0: /* LOCK */
954 case 0xf2: /* REPNE/REPNZ */
955 c->rep_prefix = REPNE_PREFIX;
957 case 0xf3: /* REP/REPE/REPZ */
958 c->rep_prefix = REPE_PREFIX;
964 /* Any legacy prefix after a REX prefix nullifies its effect. */
973 if (c->rex_prefix & 8)
974 c->op_bytes = 8; /* REX.W */
976 /* Opcode byte(s). */
977 c->d = opcode_table[c->b];
979 /* Two-byte opcode? */
982 c->b = insn_fetch(u8, 1, c->eip);
983 c->d = twobyte_table[c->b];
987 if (mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
988 kvm_report_emulation_failure(ctxt->vcpu, "invalid x86/64 instruction");;
993 group = c->d & GroupMask;
994 c->modrm = insn_fetch(u8, 1, c->eip);
997 group = (group << 3) + ((c->modrm >> 3) & 7);
998 if ((c->d & GroupDual) && (c->modrm >> 6) == 3)
999 c->d = group2_table[group];
1001 c->d = group_table[group];
1006 DPRINTF("Cannot emulate %02x\n", c->b);
1010 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
1013 /* ModRM and SIB bytes. */
1015 rc = decode_modrm(ctxt, ops);
1016 else if (c->d & MemAbs)
1017 rc = decode_abs(ctxt, ops);
1021 if (!c->has_seg_override)
1022 set_seg_override(c, VCPU_SREG_DS);
1024 if (!(!c->twobyte && c->b == 0x8d))
1025 c->modrm_ea += seg_override_base(ctxt, c);
1027 if (c->ad_bytes != 8)
1028 c->modrm_ea = (u32)c->modrm_ea;
1030 * Decode and fetch the source operand: register, memory
1033 switch (c->d & SrcMask) {
1037 decode_register_operand(&c->src, c, 0);
1046 c->src.bytes = (c->d & ByteOp) ? 1 :
1048 /* Don't fetch the address for invlpg: it could be unmapped. */
1049 if (c->twobyte && c->b == 0x01 && c->modrm_reg == 7)
1053 * For instructions with a ModR/M byte, switch to register
1054 * access if Mod = 3.
1056 if ((c->d & ModRM) && c->modrm_mod == 3) {
1057 c->src.type = OP_REG;
1058 c->src.val = c->modrm_val;
1059 c->src.ptr = c->modrm_ptr;
1062 c->src.type = OP_MEM;
1066 c->src.type = OP_IMM;
1067 c->src.ptr = (unsigned long *)c->eip;
1068 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1069 if (c->src.bytes == 8)
1071 /* NB. Immediates are sign-extended as necessary. */
1072 switch (c->src.bytes) {
1074 c->src.val = insn_fetch(s8, 1, c->eip);
1077 c->src.val = insn_fetch(s16, 2, c->eip);
1080 c->src.val = insn_fetch(s32, 4, c->eip);
1083 if ((c->d & SrcMask) == SrcImmU) {
1084 switch (c->src.bytes) {
1089 c->src.val &= 0xffff;
1092 c->src.val &= 0xffffffff;
1099 c->src.type = OP_IMM;
1100 c->src.ptr = (unsigned long *)c->eip;
1102 if ((c->d & SrcMask) == SrcImmByte)
1103 c->src.val = insn_fetch(s8, 1, c->eip);
1105 c->src.val = insn_fetch(u8, 1, c->eip);
1114 * Decode and fetch the second source operand: register, memory
1117 switch (c->d & Src2Mask) {
1122 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
1125 c->src2.type = OP_IMM;
1126 c->src2.ptr = (unsigned long *)c->eip;
1128 c->src2.val = insn_fetch(u8, 1, c->eip);
1131 c->src2.type = OP_IMM;
1132 c->src2.ptr = (unsigned long *)c->eip;
1134 c->src2.val = insn_fetch(u16, 2, c->eip);
1142 /* Decode and fetch the destination operand: register or memory. */
1143 switch (c->d & DstMask) {
1145 /* Special instructions do their own operand decoding. */
1148 decode_register_operand(&c->dst, c,
1149 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
1152 if ((c->d & ModRM) && c->modrm_mod == 3) {
1153 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1154 c->dst.type = OP_REG;
1155 c->dst.val = c->dst.orig_val = c->modrm_val;
1156 c->dst.ptr = c->modrm_ptr;
1159 c->dst.type = OP_MEM;
1162 c->dst.type = OP_REG;
1163 c->dst.bytes = c->op_bytes;
1164 c->dst.ptr = &c->regs[VCPU_REGS_RAX];
1165 switch (c->op_bytes) {
1167 c->dst.val = *(u8 *)c->dst.ptr;
1170 c->dst.val = *(u16 *)c->dst.ptr;
1173 c->dst.val = *(u32 *)c->dst.ptr;
1176 c->dst.orig_val = c->dst.val;
1180 if (c->rip_relative)
1181 c->modrm_ea += c->eip;
1184 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
1187 static inline void emulate_push(struct x86_emulate_ctxt *ctxt)
1189 struct decode_cache *c = &ctxt->decode;
1191 c->dst.type = OP_MEM;
1192 c->dst.bytes = c->op_bytes;
1193 c->dst.val = c->src.val;
1194 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1195 c->dst.ptr = (void *) register_address(c, ss_base(ctxt),
1196 c->regs[VCPU_REGS_RSP]);
1199 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1200 struct x86_emulate_ops *ops,
1201 void *dest, int len)
1203 struct decode_cache *c = &ctxt->decode;
1206 rc = ops->read_emulated(register_address(c, ss_base(ctxt),
1207 c->regs[VCPU_REGS_RSP]),
1208 dest, len, ctxt->vcpu);
1209 if (rc != X86EMUL_CONTINUE)
1212 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1216 static void emulate_push_sreg(struct x86_emulate_ctxt *ctxt, int seg)
1218 struct decode_cache *c = &ctxt->decode;
1219 struct kvm_segment segment;
1221 kvm_x86_ops->get_segment(ctxt->vcpu, &segment, seg);
1223 c->src.val = segment.selector;
1227 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1228 struct x86_emulate_ops *ops, int seg)
1230 struct decode_cache *c = &ctxt->decode;
1231 unsigned long selector;
1234 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1238 rc = kvm_load_segment_descriptor(ctxt->vcpu, (u16)selector, 1, seg);
1242 static void emulate_pusha(struct x86_emulate_ctxt *ctxt)
1244 struct decode_cache *c = &ctxt->decode;
1245 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1246 int reg = VCPU_REGS_RAX;
1248 while (reg <= VCPU_REGS_RDI) {
1249 (reg == VCPU_REGS_RSP) ?
1250 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1257 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1258 struct x86_emulate_ops *ops)
1260 struct decode_cache *c = &ctxt->decode;
1262 int reg = VCPU_REGS_RDI;
1264 while (reg >= VCPU_REGS_RAX) {
1265 if (reg == VCPU_REGS_RSP) {
1266 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1271 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1279 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1280 struct x86_emulate_ops *ops)
1282 struct decode_cache *c = &ctxt->decode;
1285 rc = emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1291 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1293 struct decode_cache *c = &ctxt->decode;
1294 switch (c->modrm_reg) {
1296 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1299 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1302 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1305 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1307 case 4: /* sal/shl */
1308 case 6: /* sal/shl */
1309 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1312 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1315 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1320 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1321 struct x86_emulate_ops *ops)
1323 struct decode_cache *c = &ctxt->decode;
1326 switch (c->modrm_reg) {
1327 case 0 ... 1: /* test */
1328 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1331 c->dst.val = ~c->dst.val;
1334 emulate_1op("neg", c->dst, ctxt->eflags);
1337 DPRINTF("Cannot emulate %02x\n", c->b);
1338 rc = X86EMUL_UNHANDLEABLE;
1344 static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
1345 struct x86_emulate_ops *ops)
1347 struct decode_cache *c = &ctxt->decode;
1349 switch (c->modrm_reg) {
1351 emulate_1op("inc", c->dst, ctxt->eflags);
1354 emulate_1op("dec", c->dst, ctxt->eflags);
1356 case 2: /* call near abs */ {
1359 c->eip = c->src.val;
1360 c->src.val = old_eip;
1364 case 4: /* jmp abs */
1365 c->eip = c->src.val;
1374 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1375 struct x86_emulate_ops *ops,
1376 unsigned long memop)
1378 struct decode_cache *c = &ctxt->decode;
1382 rc = ops->read_emulated(memop, &old, 8, ctxt->vcpu);
1383 if (rc != X86EMUL_CONTINUE)
1386 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1387 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1389 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1390 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1391 ctxt->eflags &= ~EFLG_ZF;
1394 new = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1395 (u32) c->regs[VCPU_REGS_RBX];
1397 rc = ops->cmpxchg_emulated(memop, &old, &new, 8, ctxt->vcpu);
1398 if (rc != X86EMUL_CONTINUE)
1400 ctxt->eflags |= EFLG_ZF;
1405 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1406 struct x86_emulate_ops *ops)
1408 struct decode_cache *c = &ctxt->decode;
1412 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1415 if (c->op_bytes == 4)
1416 c->eip = (u32)c->eip;
1417 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1420 rc = kvm_load_segment_descriptor(ctxt->vcpu, (u16)cs, 1, VCPU_SREG_CS);
1424 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1425 struct x86_emulate_ops *ops)
1428 struct decode_cache *c = &ctxt->decode;
1430 switch (c->dst.type) {
1432 /* The 4-byte case *is* correct:
1433 * in 64-bit mode we zero-extend.
1435 switch (c->dst.bytes) {
1437 *(u8 *)c->dst.ptr = (u8)c->dst.val;
1440 *(u16 *)c->dst.ptr = (u16)c->dst.val;
1443 *c->dst.ptr = (u32)c->dst.val;
1444 break; /* 64b: zero-ext */
1446 *c->dst.ptr = c->dst.val;
1452 rc = ops->cmpxchg_emulated(
1453 (unsigned long)c->dst.ptr,
1459 rc = ops->write_emulated(
1460 (unsigned long)c->dst.ptr,
1464 if (rc != X86EMUL_CONTINUE)
1476 static void toggle_interruptibility(struct x86_emulate_ctxt *ctxt, u32 mask)
1478 u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(ctxt->vcpu, mask);
1480 * an sti; sti; sequence only disable interrupts for the first
1481 * instruction. So, if the last instruction, be it emulated or
1482 * not, left the system with the INT_STI flag enabled, it
1483 * means that the last instruction is an sti. We should not
1484 * leave the flag on in this case. The same goes for mov ss
1486 if (!(int_shadow & mask))
1487 ctxt->interruptibility = mask;
1491 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1492 struct kvm_segment *cs, struct kvm_segment *ss)
1494 memset(cs, 0, sizeof(struct kvm_segment));
1495 kvm_x86_ops->get_segment(ctxt->vcpu, cs, VCPU_SREG_CS);
1496 memset(ss, 0, sizeof(struct kvm_segment));
1498 cs->l = 0; /* will be adjusted later */
1499 cs->base = 0; /* flat segment */
1500 cs->g = 1; /* 4kb granularity */
1501 cs->limit = 0xffffffff; /* 4GB limit */
1502 cs->type = 0x0b; /* Read, Execute, Accessed */
1504 cs->dpl = 0; /* will be adjusted later */
1509 ss->base = 0; /* flat segment */
1510 ss->limit = 0xffffffff; /* 4GB limit */
1511 ss->g = 1; /* 4kb granularity */
1513 ss->type = 0x03; /* Read/Write, Accessed */
1514 ss->db = 1; /* 32bit stack segment */
1520 emulate_syscall(struct x86_emulate_ctxt *ctxt)
1522 struct decode_cache *c = &ctxt->decode;
1523 struct kvm_segment cs, ss;
1526 /* syscall is not available in real mode */
1527 if (c->lock_prefix || ctxt->mode == X86EMUL_MODE_REAL
1528 || !is_protmode(ctxt->vcpu))
1531 setup_syscalls_segments(ctxt, &cs, &ss);
1533 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1535 cs.selector = (u16)(msr_data & 0xfffc);
1536 ss.selector = (u16)(msr_data + 8);
1538 if (is_long_mode(ctxt->vcpu)) {
1542 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1543 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1545 c->regs[VCPU_REGS_RCX] = c->eip;
1546 if (is_long_mode(ctxt->vcpu)) {
1547 #ifdef CONFIG_X86_64
1548 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1550 kvm_x86_ops->get_msr(ctxt->vcpu,
1551 ctxt->mode == X86EMUL_MODE_PROT64 ?
1552 MSR_LSTAR : MSR_CSTAR, &msr_data);
1555 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
1556 ctxt->eflags &= ~(msr_data | EFLG_RF);
1560 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1561 c->eip = (u32)msr_data;
1563 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1570 emulate_sysenter(struct x86_emulate_ctxt *ctxt)
1572 struct decode_cache *c = &ctxt->decode;
1573 struct kvm_segment cs, ss;
1576 /* inject #UD if LOCK prefix is used */
1580 /* inject #GP if in real mode or paging is disabled */
1581 if (ctxt->mode == X86EMUL_MODE_REAL || !is_protmode(ctxt->vcpu)) {
1582 kvm_inject_gp(ctxt->vcpu, 0);
1586 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1587 * Therefore, we inject an #UD.
1589 if (ctxt->mode == X86EMUL_MODE_PROT64)
1592 setup_syscalls_segments(ctxt, &cs, &ss);
1594 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1595 switch (ctxt->mode) {
1596 case X86EMUL_MODE_PROT32:
1597 if ((msr_data & 0xfffc) == 0x0) {
1598 kvm_inject_gp(ctxt->vcpu, 0);
1602 case X86EMUL_MODE_PROT64:
1603 if (msr_data == 0x0) {
1604 kvm_inject_gp(ctxt->vcpu, 0);
1610 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1611 cs.selector = (u16)msr_data;
1612 cs.selector &= ~SELECTOR_RPL_MASK;
1613 ss.selector = cs.selector + 8;
1614 ss.selector &= ~SELECTOR_RPL_MASK;
1615 if (ctxt->mode == X86EMUL_MODE_PROT64
1616 || is_long_mode(ctxt->vcpu)) {
1621 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1622 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1624 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
1627 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
1628 c->regs[VCPU_REGS_RSP] = msr_data;
1634 emulate_sysexit(struct x86_emulate_ctxt *ctxt)
1636 struct decode_cache *c = &ctxt->decode;
1637 struct kvm_segment cs, ss;
1641 /* inject #UD if LOCK prefix is used */
1645 /* inject #GP if in real mode or paging is disabled */
1646 if (ctxt->mode == X86EMUL_MODE_REAL || !is_protmode(ctxt->vcpu)) {
1647 kvm_inject_gp(ctxt->vcpu, 0);
1651 /* sysexit must be called from CPL 0 */
1652 if (kvm_x86_ops->get_cpl(ctxt->vcpu) != 0) {
1653 kvm_inject_gp(ctxt->vcpu, 0);
1657 setup_syscalls_segments(ctxt, &cs, &ss);
1659 if ((c->rex_prefix & 0x8) != 0x0)
1660 usermode = X86EMUL_MODE_PROT64;
1662 usermode = X86EMUL_MODE_PROT32;
1666 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1668 case X86EMUL_MODE_PROT32:
1669 cs.selector = (u16)(msr_data + 16);
1670 if ((msr_data & 0xfffc) == 0x0) {
1671 kvm_inject_gp(ctxt->vcpu, 0);
1674 ss.selector = (u16)(msr_data + 24);
1676 case X86EMUL_MODE_PROT64:
1677 cs.selector = (u16)(msr_data + 32);
1678 if (msr_data == 0x0) {
1679 kvm_inject_gp(ctxt->vcpu, 0);
1682 ss.selector = cs.selector + 8;
1687 cs.selector |= SELECTOR_RPL_MASK;
1688 ss.selector |= SELECTOR_RPL_MASK;
1690 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1691 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1693 c->eip = ctxt->vcpu->arch.regs[VCPU_REGS_RDX];
1694 c->regs[VCPU_REGS_RSP] = ctxt->vcpu->arch.regs[VCPU_REGS_RCX];
1700 x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1702 unsigned long memop = 0;
1704 unsigned long saved_eip = 0;
1705 struct decode_cache *c = &ctxt->decode;
1710 ctxt->interruptibility = 0;
1712 /* Shadow copy of register state. Committed on successful emulation.
1713 * NOTE: we can copy them from vcpu as x86_decode_insn() doesn't
1717 memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs);
1720 if (((c->d & ModRM) && (c->modrm_mod != 3)) || (c->d & MemAbs))
1721 memop = c->modrm_ea;
1723 if (c->rep_prefix && (c->d & String)) {
1724 /* All REP prefixes have the same first termination condition */
1725 if (c->regs[VCPU_REGS_RCX] == 0) {
1726 kvm_rip_write(ctxt->vcpu, c->eip);
1729 /* The second termination condition only applies for REPE
1730 * and REPNE. Test if the repeat string operation prefix is
1731 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
1732 * corresponding termination condition according to:
1733 * - if REPE/REPZ and ZF = 0 then done
1734 * - if REPNE/REPNZ and ZF = 1 then done
1736 if ((c->b == 0xa6) || (c->b == 0xa7) ||
1737 (c->b == 0xae) || (c->b == 0xaf)) {
1738 if ((c->rep_prefix == REPE_PREFIX) &&
1739 ((ctxt->eflags & EFLG_ZF) == 0)) {
1740 kvm_rip_write(ctxt->vcpu, c->eip);
1743 if ((c->rep_prefix == REPNE_PREFIX) &&
1744 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF)) {
1745 kvm_rip_write(ctxt->vcpu, c->eip);
1749 c->regs[VCPU_REGS_RCX]--;
1750 c->eip = kvm_rip_read(ctxt->vcpu);
1753 if (c->src.type == OP_MEM) {
1754 c->src.ptr = (unsigned long *)memop;
1756 rc = ops->read_emulated((unsigned long)c->src.ptr,
1760 if (rc != X86EMUL_CONTINUE)
1762 c->src.orig_val = c->src.val;
1765 if ((c->d & DstMask) == ImplicitOps)
1769 if (c->dst.type == OP_MEM) {
1770 c->dst.ptr = (unsigned long *)memop;
1771 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1774 unsigned long mask = ~(c->dst.bytes * 8 - 1);
1776 c->dst.ptr = (void *)c->dst.ptr +
1777 (c->src.val & mask) / 8;
1779 if (!(c->d & Mov)) {
1780 /* optimisation - avoid slow emulated read */
1781 rc = ops->read_emulated((unsigned long)c->dst.ptr,
1785 if (rc != X86EMUL_CONTINUE)
1789 c->dst.orig_val = c->dst.val;
1799 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
1801 case 0x06: /* push es */
1802 emulate_push_sreg(ctxt, VCPU_SREG_ES);
1804 case 0x07: /* pop es */
1805 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
1811 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
1813 case 0x0e: /* push cs */
1814 emulate_push_sreg(ctxt, VCPU_SREG_CS);
1818 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
1820 case 0x16: /* push ss */
1821 emulate_push_sreg(ctxt, VCPU_SREG_SS);
1823 case 0x17: /* pop ss */
1824 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
1830 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
1832 case 0x1e: /* push ds */
1833 emulate_push_sreg(ctxt, VCPU_SREG_DS);
1835 case 0x1f: /* pop ds */
1836 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
1842 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
1846 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
1850 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
1854 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
1856 case 0x40 ... 0x47: /* inc r16/r32 */
1857 emulate_1op("inc", c->dst, ctxt->eflags);
1859 case 0x48 ... 0x4f: /* dec r16/r32 */
1860 emulate_1op("dec", c->dst, ctxt->eflags);
1862 case 0x50 ... 0x57: /* push reg */
1865 case 0x58 ... 0x5f: /* pop reg */
1867 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
1871 case 0x60: /* pusha */
1872 emulate_pusha(ctxt);
1874 case 0x61: /* popa */
1875 rc = emulate_popa(ctxt, ops);
1879 case 0x63: /* movsxd */
1880 if (ctxt->mode != X86EMUL_MODE_PROT64)
1881 goto cannot_emulate;
1882 c->dst.val = (s32) c->src.val;
1884 case 0x68: /* push imm */
1885 case 0x6a: /* push imm8 */
1888 case 0x6c: /* insb */
1889 case 0x6d: /* insw/insd */
1890 if (kvm_emulate_pio_string(ctxt->vcpu,
1892 (c->d & ByteOp) ? 1 : c->op_bytes,
1894 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1,
1895 (ctxt->eflags & EFLG_DF),
1896 register_address(c, es_base(ctxt),
1897 c->regs[VCPU_REGS_RDI]),
1899 c->regs[VCPU_REGS_RDX]) == 0) {
1904 case 0x6e: /* outsb */
1905 case 0x6f: /* outsw/outsd */
1906 if (kvm_emulate_pio_string(ctxt->vcpu,
1908 (c->d & ByteOp) ? 1 : c->op_bytes,
1910 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1,
1911 (ctxt->eflags & EFLG_DF),
1913 seg_override_base(ctxt, c),
1914 c->regs[VCPU_REGS_RSI]),
1916 c->regs[VCPU_REGS_RDX]) == 0) {
1921 case 0x70 ... 0x7f: /* jcc (short) */
1922 if (test_cc(c->b, ctxt->eflags))
1923 jmp_rel(c, c->src.val);
1925 case 0x80 ... 0x83: /* Grp1 */
1926 switch (c->modrm_reg) {
1946 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1948 case 0x86 ... 0x87: /* xchg */
1950 /* Write back the register source. */
1951 switch (c->dst.bytes) {
1953 *(u8 *) c->src.ptr = (u8) c->dst.val;
1956 *(u16 *) c->src.ptr = (u16) c->dst.val;
1959 *c->src.ptr = (u32) c->dst.val;
1960 break; /* 64b reg: zero-extend */
1962 *c->src.ptr = c->dst.val;
1966 * Write back the memory destination with implicit LOCK
1969 c->dst.val = c->src.val;
1972 case 0x88 ... 0x8b: /* mov */
1974 case 0x8c: { /* mov r/m, sreg */
1975 struct kvm_segment segreg;
1977 if (c->modrm_reg <= 5)
1978 kvm_get_segment(ctxt->vcpu, &segreg, c->modrm_reg);
1980 printk(KERN_INFO "0x8c: Invalid segreg in modrm byte 0x%02x\n",
1982 goto cannot_emulate;
1984 c->dst.val = segreg.selector;
1987 case 0x8d: /* lea r16/r32, m */
1988 c->dst.val = c->modrm_ea;
1990 case 0x8e: { /* mov seg, r/m16 */
1996 if (c->modrm_reg == VCPU_SREG_SS)
1997 toggle_interruptibility(ctxt, X86_SHADOW_INT_MOV_SS);
1999 if (c->modrm_reg <= 5) {
2000 type_bits = (c->modrm_reg == 1) ? 9 : 1;
2001 err = kvm_load_segment_descriptor(ctxt->vcpu, sel,
2002 type_bits, c->modrm_reg);
2004 printk(KERN_INFO "Invalid segreg in modrm byte 0x%02x\n",
2006 goto cannot_emulate;
2010 goto cannot_emulate;
2012 c->dst.type = OP_NONE; /* Disable writeback. */
2015 case 0x8f: /* pop (sole member of Grp1a) */
2016 rc = emulate_grp1a(ctxt, ops);
2020 case 0x90: /* nop / xchg r8,rax */
2021 if (!(c->rex_prefix & 1)) { /* nop */
2022 c->dst.type = OP_NONE;
2025 case 0x91 ... 0x97: /* xchg reg,rax */
2026 c->src.type = c->dst.type = OP_REG;
2027 c->src.bytes = c->dst.bytes = c->op_bytes;
2028 c->src.ptr = (unsigned long *) &c->regs[VCPU_REGS_RAX];
2029 c->src.val = *(c->src.ptr);
2031 case 0x9c: /* pushf */
2032 c->src.val = (unsigned long) ctxt->eflags;
2035 case 0x9d: /* popf */
2036 c->dst.type = OP_REG;
2037 c->dst.ptr = (unsigned long *) &ctxt->eflags;
2038 c->dst.bytes = c->op_bytes;
2039 goto pop_instruction;
2040 case 0xa0 ... 0xa1: /* mov */
2041 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2042 c->dst.val = c->src.val;
2044 case 0xa2 ... 0xa3: /* mov */
2045 c->dst.val = (unsigned long)c->regs[VCPU_REGS_RAX];
2047 case 0xa4 ... 0xa5: /* movs */
2048 c->dst.type = OP_MEM;
2049 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2050 c->dst.ptr = (unsigned long *)register_address(c,
2052 c->regs[VCPU_REGS_RDI]);
2053 rc = ops->read_emulated(register_address(c,
2054 seg_override_base(ctxt, c),
2055 c->regs[VCPU_REGS_RSI]),
2057 c->dst.bytes, ctxt->vcpu);
2058 if (rc != X86EMUL_CONTINUE)
2060 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2061 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2063 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2064 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2067 case 0xa6 ... 0xa7: /* cmps */
2068 c->src.type = OP_NONE; /* Disable writeback. */
2069 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2070 c->src.ptr = (unsigned long *)register_address(c,
2071 seg_override_base(ctxt, c),
2072 c->regs[VCPU_REGS_RSI]);
2073 rc = ops->read_emulated((unsigned long)c->src.ptr,
2077 if (rc != X86EMUL_CONTINUE)
2080 c->dst.type = OP_NONE; /* Disable writeback. */
2081 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2082 c->dst.ptr = (unsigned long *)register_address(c,
2084 c->regs[VCPU_REGS_RDI]);
2085 rc = ops->read_emulated((unsigned long)c->dst.ptr,
2089 if (rc != X86EMUL_CONTINUE)
2092 DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.ptr, c->dst.ptr);
2094 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2096 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2097 (ctxt->eflags & EFLG_DF) ? -c->src.bytes
2099 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2100 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2104 case 0xaa ... 0xab: /* stos */
2105 c->dst.type = OP_MEM;
2106 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2107 c->dst.ptr = (unsigned long *)register_address(c,
2109 c->regs[VCPU_REGS_RDI]);
2110 c->dst.val = c->regs[VCPU_REGS_RAX];
2111 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2112 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2115 case 0xac ... 0xad: /* lods */
2116 c->dst.type = OP_REG;
2117 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2118 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2119 rc = ops->read_emulated(register_address(c,
2120 seg_override_base(ctxt, c),
2121 c->regs[VCPU_REGS_RSI]),
2125 if (rc != X86EMUL_CONTINUE)
2127 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2128 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2131 case 0xae ... 0xaf: /* scas */
2132 DPRINTF("Urk! I don't handle SCAS.\n");
2133 goto cannot_emulate;
2134 case 0xb0 ... 0xbf: /* mov r, imm */
2139 case 0xc3: /* ret */
2140 c->dst.type = OP_REG;
2141 c->dst.ptr = &c->eip;
2142 c->dst.bytes = c->op_bytes;
2143 goto pop_instruction;
2144 case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */
2146 c->dst.val = c->src.val;
2148 case 0xcb: /* ret far */
2149 rc = emulate_ret_far(ctxt, ops);
2153 case 0xd0 ... 0xd1: /* Grp2 */
2157 case 0xd2 ... 0xd3: /* Grp2 */
2158 c->src.val = c->regs[VCPU_REGS_RCX];
2161 case 0xe4: /* inb */
2166 case 0xe6: /* outb */
2167 case 0xe7: /* out */
2171 case 0xe8: /* call (near) */ {
2172 long int rel = c->src.val;
2173 c->src.val = (unsigned long) c->eip;
2178 case 0xe9: /* jmp rel */
2180 case 0xea: /* jmp far */
2181 if (kvm_load_segment_descriptor(ctxt->vcpu, c->src2.val, 9,
2182 VCPU_SREG_CS) < 0) {
2183 DPRINTF("jmp far: Failed to load CS descriptor\n");
2184 goto cannot_emulate;
2187 c->eip = c->src.val;
2190 jmp: /* jmp rel short */
2191 jmp_rel(c, c->src.val);
2192 c->dst.type = OP_NONE; /* Disable writeback. */
2194 case 0xec: /* in al,dx */
2195 case 0xed: /* in (e/r)ax,dx */
2196 port = c->regs[VCPU_REGS_RDX];
2199 case 0xee: /* out al,dx */
2200 case 0xef: /* out (e/r)ax,dx */
2201 port = c->regs[VCPU_REGS_RDX];
2203 do_io: if (kvm_emulate_pio(ctxt->vcpu, io_dir_in,
2204 (c->d & ByteOp) ? 1 : c->op_bytes,
2207 goto cannot_emulate;
2210 case 0xf4: /* hlt */
2211 ctxt->vcpu->arch.halt_request = 1;
2213 case 0xf5: /* cmc */
2214 /* complement carry flag from eflags reg */
2215 ctxt->eflags ^= EFLG_CF;
2216 c->dst.type = OP_NONE; /* Disable writeback. */
2218 case 0xf6 ... 0xf7: /* Grp3 */
2219 rc = emulate_grp3(ctxt, ops);
2223 case 0xf8: /* clc */
2224 ctxt->eflags &= ~EFLG_CF;
2225 c->dst.type = OP_NONE; /* Disable writeback. */
2227 case 0xfa: /* cli */
2228 ctxt->eflags &= ~X86_EFLAGS_IF;
2229 c->dst.type = OP_NONE; /* Disable writeback. */
2231 case 0xfb: /* sti */
2232 toggle_interruptibility(ctxt, X86_SHADOW_INT_STI);
2233 ctxt->eflags |= X86_EFLAGS_IF;
2234 c->dst.type = OP_NONE; /* Disable writeback. */
2236 case 0xfc: /* cld */
2237 ctxt->eflags &= ~EFLG_DF;
2238 c->dst.type = OP_NONE; /* Disable writeback. */
2240 case 0xfd: /* std */
2241 ctxt->eflags |= EFLG_DF;
2242 c->dst.type = OP_NONE; /* Disable writeback. */
2244 case 0xfe ... 0xff: /* Grp4/Grp5 */
2245 rc = emulate_grp45(ctxt, ops);
2252 rc = writeback(ctxt, ops);
2256 /* Commit shadow register state. */
2257 memcpy(ctxt->vcpu->arch.regs, c->regs, sizeof c->regs);
2258 kvm_rip_write(ctxt->vcpu, c->eip);
2261 if (rc == X86EMUL_UNHANDLEABLE) {
2269 case 0x01: /* lgdt, lidt, lmsw */
2270 switch (c->modrm_reg) {
2272 unsigned long address;
2274 case 0: /* vmcall */
2275 if (c->modrm_mod != 3 || c->modrm_rm != 1)
2276 goto cannot_emulate;
2278 rc = kvm_fix_hypercall(ctxt->vcpu);
2282 /* Let the processor re-execute the fixed hypercall */
2283 c->eip = kvm_rip_read(ctxt->vcpu);
2284 /* Disable writeback. */
2285 c->dst.type = OP_NONE;
2288 rc = read_descriptor(ctxt, ops, c->src.ptr,
2289 &size, &address, c->op_bytes);
2292 realmode_lgdt(ctxt->vcpu, size, address);
2293 /* Disable writeback. */
2294 c->dst.type = OP_NONE;
2296 case 3: /* lidt/vmmcall */
2297 if (c->modrm_mod == 3) {
2298 switch (c->modrm_rm) {
2300 rc = kvm_fix_hypercall(ctxt->vcpu);
2305 goto cannot_emulate;
2308 rc = read_descriptor(ctxt, ops, c->src.ptr,
2313 realmode_lidt(ctxt->vcpu, size, address);
2315 /* Disable writeback. */
2316 c->dst.type = OP_NONE;
2320 c->dst.val = realmode_get_cr(ctxt->vcpu, 0);
2323 realmode_lmsw(ctxt->vcpu, (u16)c->src.val,
2325 c->dst.type = OP_NONE;
2328 emulate_invlpg(ctxt->vcpu, memop);
2329 /* Disable writeback. */
2330 c->dst.type = OP_NONE;
2333 goto cannot_emulate;
2336 case 0x05: /* syscall */
2337 if (emulate_syscall(ctxt) == -1)
2338 goto cannot_emulate;
2343 emulate_clts(ctxt->vcpu);
2344 c->dst.type = OP_NONE;
2346 case 0x08: /* invd */
2347 case 0x09: /* wbinvd */
2348 case 0x0d: /* GrpP (prefetch) */
2349 case 0x18: /* Grp16 (prefetch/nop) */
2350 c->dst.type = OP_NONE;
2352 case 0x20: /* mov cr, reg */
2353 if (c->modrm_mod != 3)
2354 goto cannot_emulate;
2355 c->regs[c->modrm_rm] =
2356 realmode_get_cr(ctxt->vcpu, c->modrm_reg);
2357 c->dst.type = OP_NONE; /* no writeback */
2359 case 0x21: /* mov from dr to reg */
2360 if (c->modrm_mod != 3)
2361 goto cannot_emulate;
2362 rc = emulator_get_dr(ctxt, c->modrm_reg, &c->regs[c->modrm_rm]);
2364 goto cannot_emulate;
2365 c->dst.type = OP_NONE; /* no writeback */
2367 case 0x22: /* mov reg, cr */
2368 if (c->modrm_mod != 3)
2369 goto cannot_emulate;
2370 realmode_set_cr(ctxt->vcpu,
2371 c->modrm_reg, c->modrm_val, &ctxt->eflags);
2372 c->dst.type = OP_NONE;
2374 case 0x23: /* mov from reg to dr */
2375 if (c->modrm_mod != 3)
2376 goto cannot_emulate;
2377 rc = emulator_set_dr(ctxt, c->modrm_reg,
2378 c->regs[c->modrm_rm]);
2380 goto cannot_emulate;
2381 c->dst.type = OP_NONE; /* no writeback */
2385 msr_data = (u32)c->regs[VCPU_REGS_RAX]
2386 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
2387 rc = kvm_set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data);
2389 kvm_inject_gp(ctxt->vcpu, 0);
2390 c->eip = kvm_rip_read(ctxt->vcpu);
2392 rc = X86EMUL_CONTINUE;
2393 c->dst.type = OP_NONE;
2397 rc = kvm_get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data);
2399 kvm_inject_gp(ctxt->vcpu, 0);
2400 c->eip = kvm_rip_read(ctxt->vcpu);
2402 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
2403 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
2405 rc = X86EMUL_CONTINUE;
2406 c->dst.type = OP_NONE;
2408 case 0x34: /* sysenter */
2409 if (emulate_sysenter(ctxt) == -1)
2410 goto cannot_emulate;
2414 case 0x35: /* sysexit */
2415 if (emulate_sysexit(ctxt) == -1)
2416 goto cannot_emulate;
2420 case 0x40 ... 0x4f: /* cmov */
2421 c->dst.val = c->dst.orig_val = c->src.val;
2422 if (!test_cc(c->b, ctxt->eflags))
2423 c->dst.type = OP_NONE; /* no writeback */
2425 case 0x80 ... 0x8f: /* jnz rel, etc*/
2426 if (test_cc(c->b, ctxt->eflags))
2427 jmp_rel(c, c->src.val);
2428 c->dst.type = OP_NONE;
2430 case 0xa0: /* push fs */
2431 emulate_push_sreg(ctxt, VCPU_SREG_FS);
2433 case 0xa1: /* pop fs */
2434 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
2440 c->dst.type = OP_NONE;
2441 /* only subword offset */
2442 c->src.val &= (c->dst.bytes << 3) - 1;
2443 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
2445 case 0xa4: /* shld imm8, r, r/m */
2446 case 0xa5: /* shld cl, r, r/m */
2447 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
2449 case 0xa8: /* push gs */
2450 emulate_push_sreg(ctxt, VCPU_SREG_GS);
2452 case 0xa9: /* pop gs */
2453 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
2459 /* only subword offset */
2460 c->src.val &= (c->dst.bytes << 3) - 1;
2461 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
2463 case 0xac: /* shrd imm8, r, r/m */
2464 case 0xad: /* shrd cl, r, r/m */
2465 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
2467 case 0xae: /* clflush */
2469 case 0xb0 ... 0xb1: /* cmpxchg */
2471 * Save real source value, then compare EAX against
2474 c->src.orig_val = c->src.val;
2475 c->src.val = c->regs[VCPU_REGS_RAX];
2476 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2477 if (ctxt->eflags & EFLG_ZF) {
2478 /* Success: write back to memory. */
2479 c->dst.val = c->src.orig_val;
2481 /* Failure: write the value we saw to EAX. */
2482 c->dst.type = OP_REG;
2483 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2488 /* only subword offset */
2489 c->src.val &= (c->dst.bytes << 3) - 1;
2490 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
2492 case 0xb6 ... 0xb7: /* movzx */
2493 c->dst.bytes = c->op_bytes;
2494 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
2497 case 0xba: /* Grp8 */
2498 switch (c->modrm_reg & 3) {
2511 /* only subword offset */
2512 c->src.val &= (c->dst.bytes << 3) - 1;
2513 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
2515 case 0xbe ... 0xbf: /* movsx */
2516 c->dst.bytes = c->op_bytes;
2517 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
2520 case 0xc3: /* movnti */
2521 c->dst.bytes = c->op_bytes;
2522 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
2525 case 0xc7: /* Grp9 (cmpxchg8b) */
2526 rc = emulate_grp9(ctxt, ops, memop);
2529 c->dst.type = OP_NONE;
2535 DPRINTF("Cannot emulate %02x\n", c->b);