1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
13 * Avi Kivity <avi@qumranet.com>
14 * Yaniv Kamay <yaniv@qumranet.com>
16 * This work is licensed under the terms of the GNU GPL, version 2. See
17 * the COPYING file in the top-level directory.
19 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
25 #include <public/xen.h>
26 #define DPRINTF(_f, _a ...) printf(_f , ## _a)
28 #include <linux/kvm_host.h>
29 #include "kvm_cache_regs.h"
30 #define DPRINTF(x...) do {} while (0)
32 #include <linux/module.h>
33 #include <asm/kvm_emulate.h>
35 #include "mmu.h" /* for is_long_mode() */
38 * Opcode effective-address decode tables.
39 * Note that we only emulate instructions that have at least one memory
40 * operand (excluding implicit stack references). We assume that stack
41 * references and instruction fetches will never occur in special memory
42 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
46 /* Operand sizes: 8-bit operands or specified/overridden size. */
47 #define ByteOp (1<<0) /* 8-bit operands. */
48 /* Destination operand type. */
49 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
50 #define DstReg (2<<1) /* Register operand. */
51 #define DstMem (3<<1) /* Memory operand. */
52 #define DstAcc (4<<1) /* Destination Accumulator */
53 #define DstMask (7<<1)
54 /* Source operand type. */
55 #define SrcNone (0<<4) /* No source operand. */
56 #define SrcImplicit (0<<4) /* Source operand is implicit in the opcode. */
57 #define SrcReg (1<<4) /* Register operand. */
58 #define SrcMem (2<<4) /* Memory operand. */
59 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
60 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
61 #define SrcImm (5<<4) /* Immediate operand. */
62 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
63 #define SrcOne (7<<4) /* Implied '1' */
64 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
65 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
66 #define SrcMask (0xf<<4)
67 /* Generic ModRM decode. */
69 /* Destination is only written; never read. */
72 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
73 #define String (1<<12) /* String instruction (rep capable) */
74 #define Stack (1<<13) /* Stack instruction (push/pop) */
75 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
76 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
77 #define GroupMask 0xff /* Group number stored in bits 0:7 */
80 /* Source 2 operand type */
81 #define Src2None (0<<29)
82 #define Src2CL (1<<29)
83 #define Src2ImmByte (2<<29)
84 #define Src2One (3<<29)
85 #define Src2Imm16 (4<<29)
86 #define Src2Mask (7<<29)
89 Group1_80, Group1_81, Group1_82, Group1_83,
90 Group1A, Group3_Byte, Group3, Group4, Group5, Group7,
93 static u32 opcode_table[256] = {
95 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
96 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
97 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
98 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
100 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
101 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
102 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
103 ImplicitOps | Stack | No64, 0,
105 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
106 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
107 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
108 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
110 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
111 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
112 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
113 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
115 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
116 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
117 DstAcc | SrcImmByte, DstAcc | SrcImm, 0, 0,
119 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
120 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
123 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
124 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
127 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
128 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
129 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
132 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
134 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
136 SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack,
137 SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack,
139 DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack,
140 DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack,
142 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
143 0, DstReg | SrcMem32 | ModRM | Mov /* movsxd (x86/64) */ ,
146 SrcImm | Mov | Stack, 0, SrcImmByte | Mov | Stack, 0,
147 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* insb, insw/insd */
148 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* outsb, outsw/outsd */
150 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
151 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
153 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
154 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
156 Group | Group1_80, Group | Group1_81,
157 Group | Group1_82, Group | Group1_83,
158 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
159 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
161 ByteOp | DstMem | SrcReg | ModRM | Mov, DstMem | SrcReg | ModRM | Mov,
162 ByteOp | DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
163 DstMem | SrcReg | ModRM | Mov, ModRM | DstReg,
164 DstReg | SrcMem | ModRM | Mov, Group | Group1A,
166 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
168 0, 0, SrcImm | Src2Imm16 | No64, 0,
169 ImplicitOps | Stack, ImplicitOps | Stack, 0, 0,
171 ByteOp | DstReg | SrcMem | Mov | MemAbs, DstReg | SrcMem | Mov | MemAbs,
172 ByteOp | DstMem | SrcReg | Mov | MemAbs, DstMem | SrcReg | Mov | MemAbs,
173 ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
174 ByteOp | ImplicitOps | String, ImplicitOps | String,
176 0, 0, ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
177 ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
178 ByteOp | ImplicitOps | String, ImplicitOps | String,
180 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
181 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
182 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
183 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
185 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
186 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
187 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
188 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
190 ByteOp | DstMem | SrcImm | ModRM, DstMem | SrcImmByte | ModRM,
191 0, ImplicitOps | Stack, 0, 0,
192 ByteOp | DstMem | SrcImm | ModRM | Mov, DstMem | SrcImm | ModRM | Mov,
194 0, 0, 0, ImplicitOps | Stack,
195 ImplicitOps, SrcImmByte, ImplicitOps | No64, ImplicitOps,
197 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
198 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
201 0, 0, 0, 0, 0, 0, 0, 0,
204 ByteOp | SrcImmUByte, SrcImmUByte,
205 ByteOp | SrcImmUByte, SrcImmUByte,
207 SrcImm | Stack, SrcImm | ImplicitOps,
208 SrcImmU | Src2Imm16 | No64, SrcImmByte | ImplicitOps,
209 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps,
210 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps,
213 ImplicitOps, ImplicitOps, Group | Group3_Byte, Group | Group3,
215 ImplicitOps, 0, ImplicitOps, ImplicitOps,
216 ImplicitOps, ImplicitOps, Group | Group4, Group | Group5,
219 static u32 twobyte_table[256] = {
221 0, Group | GroupDual | Group7, 0, 0, 0, ImplicitOps, ImplicitOps, 0,
222 ImplicitOps, ImplicitOps, 0, 0, 0, ImplicitOps | ModRM, 0, 0,
224 0, 0, 0, 0, 0, 0, 0, 0, ImplicitOps | ModRM, 0, 0, 0, 0, 0, 0, 0,
226 ModRM | ImplicitOps, ModRM, ModRM | ImplicitOps, ModRM, 0, 0, 0, 0,
227 0, 0, 0, 0, 0, 0, 0, 0,
229 ImplicitOps, 0, ImplicitOps, 0,
230 ImplicitOps, ImplicitOps, 0, 0,
231 0, 0, 0, 0, 0, 0, 0, 0,
233 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
234 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
235 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
236 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
238 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
239 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
240 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
241 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
243 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
245 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
247 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
249 SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm,
250 SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm,
252 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
254 ImplicitOps | Stack, ImplicitOps | Stack,
255 0, DstMem | SrcReg | ModRM | BitOp,
256 DstMem | SrcReg | Src2ImmByte | ModRM,
257 DstMem | SrcReg | Src2CL | ModRM, 0, 0,
259 ImplicitOps | Stack, ImplicitOps | Stack,
260 0, DstMem | SrcReg | ModRM | BitOp,
261 DstMem | SrcReg | Src2ImmByte | ModRM,
262 DstMem | SrcReg | Src2CL | ModRM,
265 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, 0,
266 DstMem | SrcReg | ModRM | BitOp,
267 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
268 DstReg | SrcMem16 | ModRM | Mov,
270 0, 0, DstMem | SrcImmByte | ModRM, DstMem | SrcReg | ModRM | BitOp,
271 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
272 DstReg | SrcMem16 | ModRM | Mov,
274 0, 0, 0, DstMem | SrcReg | ModRM | Mov, 0, 0, 0, ImplicitOps | ModRM,
275 0, 0, 0, 0, 0, 0, 0, 0,
277 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
279 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
281 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
284 static u32 group_table[] = {
286 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
287 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
288 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
289 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
291 DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
292 DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
293 DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
294 DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
296 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
297 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
298 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
299 ByteOp | DstMem | SrcImm | ModRM, ByteOp | DstMem | SrcImm | ModRM,
301 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
302 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
303 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
304 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM,
306 DstMem | SrcNone | ModRM | Mov | Stack, 0, 0, 0, 0, 0, 0, 0,
308 ByteOp | SrcImm | DstMem | ModRM, 0,
309 ByteOp | DstMem | SrcNone | ModRM, ByteOp | DstMem | SrcNone | ModRM,
312 DstMem | SrcImm | ModRM, 0,
313 DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM,
316 ByteOp | DstMem | SrcNone | ModRM, ByteOp | DstMem | SrcNone | ModRM,
319 DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM,
320 SrcMem | ModRM | Stack, 0,
321 SrcMem | ModRM | Stack, 0, SrcMem | ModRM | Stack, 0,
323 0, 0, ModRM | SrcMem, ModRM | SrcMem,
324 SrcNone | ModRM | DstMem | Mov, 0,
325 SrcMem16 | ModRM | Mov, SrcMem | ModRM | ByteOp,
328 static u32 group2_table[] = {
330 SrcNone | ModRM, 0, 0, SrcNone | ModRM,
331 SrcNone | ModRM | DstMem | Mov, 0,
332 SrcMem16 | ModRM | Mov, 0,
335 /* EFLAGS bit definitions. */
336 #define EFLG_VM (1<<17)
337 #define EFLG_RF (1<<16)
338 #define EFLG_OF (1<<11)
339 #define EFLG_DF (1<<10)
340 #define EFLG_IF (1<<9)
341 #define EFLG_SF (1<<7)
342 #define EFLG_ZF (1<<6)
343 #define EFLG_AF (1<<4)
344 #define EFLG_PF (1<<2)
345 #define EFLG_CF (1<<0)
348 * Instruction emulation:
349 * Most instructions are emulated directly via a fragment of inline assembly
350 * code. This allows us to save/restore EFLAGS and thus very easily pick up
351 * any modified flags.
354 #if defined(CONFIG_X86_64)
355 #define _LO32 "k" /* force 32-bit operand */
356 #define _STK "%%rsp" /* stack pointer */
357 #elif defined(__i386__)
358 #define _LO32 "" /* force 32-bit operand */
359 #define _STK "%%esp" /* stack pointer */
363 * These EFLAGS bits are restored from saved value during emulation, and
364 * any changes are written back to the saved value after emulation.
366 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
368 /* Before executing instruction: restore necessary bits in EFLAGS. */
369 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
370 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
371 "movl %"_sav",%"_LO32 _tmp"; " \
374 "movl %"_msk",%"_LO32 _tmp"; " \
375 "andl %"_LO32 _tmp",("_STK"); " \
377 "notl %"_LO32 _tmp"; " \
378 "andl %"_LO32 _tmp",("_STK"); " \
379 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
381 "orl %"_LO32 _tmp",("_STK"); " \
385 /* After executing instruction: write-back necessary bits in EFLAGS. */
386 #define _POST_EFLAGS(_sav, _msk, _tmp) \
387 /* _sav |= EFLAGS & _msk; */ \
390 "andl %"_msk",%"_LO32 _tmp"; " \
391 "orl %"_LO32 _tmp",%"_sav"; "
399 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix) \
401 __asm__ __volatile__ ( \
402 _PRE_EFLAGS("0", "4", "2") \
403 _op _suffix " %"_x"3,%1; " \
404 _POST_EFLAGS("0", "4", "2") \
405 : "=m" (_eflags), "=m" ((_dst).val), \
407 : _y ((_src).val), "i" (EFLAGS_MASK)); \
411 /* Raw emulation: instruction has two explicit operands. */
412 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
414 unsigned long _tmp; \
416 switch ((_dst).bytes) { \
418 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w"); \
421 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l"); \
424 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q")); \
429 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
431 unsigned long _tmp; \
432 switch ((_dst).bytes) { \
434 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b"); \
437 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
438 _wx, _wy, _lx, _ly, _qx, _qy); \
443 /* Source operand is byte-sized and may be restricted to just %cl. */
444 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
445 __emulate_2op(_op, _src, _dst, _eflags, \
446 "b", "c", "b", "c", "b", "c", "b", "c")
448 /* Source operand is byte, word, long or quad sized. */
449 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
450 __emulate_2op(_op, _src, _dst, _eflags, \
451 "b", "q", "w", "r", _LO32, "r", "", "r")
453 /* Source operand is word, long or quad sized. */
454 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
455 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
456 "w", "r", _LO32, "r", "", "r")
458 /* Instruction has three operands and one operand is stored in ECX register */
459 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
461 unsigned long _tmp; \
462 _type _clv = (_cl).val; \
463 _type _srcv = (_src).val; \
464 _type _dstv = (_dst).val; \
466 __asm__ __volatile__ ( \
467 _PRE_EFLAGS("0", "5", "2") \
468 _op _suffix " %4,%1 \n" \
469 _POST_EFLAGS("0", "5", "2") \
470 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
471 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
474 (_cl).val = (unsigned long) _clv; \
475 (_src).val = (unsigned long) _srcv; \
476 (_dst).val = (unsigned long) _dstv; \
479 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
481 switch ((_dst).bytes) { \
483 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
484 "w", unsigned short); \
487 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
488 "l", unsigned int); \
491 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
492 "q", unsigned long)); \
497 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
499 unsigned long _tmp; \
501 __asm__ __volatile__ ( \
502 _PRE_EFLAGS("0", "3", "2") \
503 _op _suffix " %1; " \
504 _POST_EFLAGS("0", "3", "2") \
505 : "=m" (_eflags), "+m" ((_dst).val), \
507 : "i" (EFLAGS_MASK)); \
510 /* Instruction has only one explicit operand (no source operand). */
511 #define emulate_1op(_op, _dst, _eflags) \
513 switch ((_dst).bytes) { \
514 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
515 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
516 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
517 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
521 /* Fetch next part of the instruction being emulated. */
522 #define insn_fetch(_type, _size, _eip) \
523 ({ unsigned long _x; \
524 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
531 static inline unsigned long ad_mask(struct decode_cache *c)
533 return (1UL << (c->ad_bytes << 3)) - 1;
536 /* Access/update address held in a register, based on addressing mode. */
537 static inline unsigned long
538 address_mask(struct decode_cache *c, unsigned long reg)
540 if (c->ad_bytes == sizeof(unsigned long))
543 return reg & ad_mask(c);
546 static inline unsigned long
547 register_address(struct decode_cache *c, unsigned long base, unsigned long reg)
549 return base + address_mask(c, reg);
553 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
555 if (c->ad_bytes == sizeof(unsigned long))
558 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
561 static inline void jmp_rel(struct decode_cache *c, int rel)
563 register_address_increment(c, &c->eip, rel);
566 static void set_seg_override(struct decode_cache *c, int seg)
568 c->has_seg_override = true;
569 c->seg_override = seg;
572 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
574 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
577 return kvm_x86_ops->get_segment_base(ctxt->vcpu, seg);
580 static unsigned long seg_override_base(struct x86_emulate_ctxt *ctxt,
581 struct decode_cache *c)
583 if (!c->has_seg_override)
586 return seg_base(ctxt, c->seg_override);
589 static unsigned long es_base(struct x86_emulate_ctxt *ctxt)
591 return seg_base(ctxt, VCPU_SREG_ES);
594 static unsigned long ss_base(struct x86_emulate_ctxt *ctxt)
596 return seg_base(ctxt, VCPU_SREG_SS);
599 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
600 struct x86_emulate_ops *ops,
601 unsigned long linear, u8 *dest)
603 struct fetch_cache *fc = &ctxt->decode.fetch;
607 if (linear < fc->start || linear >= fc->end) {
608 size = min(15UL, PAGE_SIZE - offset_in_page(linear));
609 rc = ops->read_std(linear, fc->data, size, ctxt->vcpu);
613 fc->end = linear + size;
615 *dest = fc->data[linear - fc->start];
619 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
620 struct x86_emulate_ops *ops,
621 unsigned long eip, void *dest, unsigned size)
625 /* x86 instructions are limited to 15 bytes. */
626 if (eip + size - ctxt->decode.eip_orig > 15)
627 return X86EMUL_UNHANDLEABLE;
628 eip += ctxt->cs_base;
630 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
638 * Given the 'reg' portion of a ModRM byte, and a register block, return a
639 * pointer into the block that addresses the relevant register.
640 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
642 static void *decode_register(u8 modrm_reg, unsigned long *regs,
647 p = ®s[modrm_reg];
648 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
649 p = (unsigned char *)®s[modrm_reg & 3] + 1;
653 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
654 struct x86_emulate_ops *ops,
656 u16 *size, unsigned long *address, int op_bytes)
663 rc = ops->read_std((unsigned long)ptr, (unsigned long *)size, 2,
667 rc = ops->read_std((unsigned long)ptr + 2, address, op_bytes,
672 static int test_cc(unsigned int condition, unsigned int flags)
676 switch ((condition & 15) >> 1) {
678 rc |= (flags & EFLG_OF);
680 case 1: /* b/c/nae */
681 rc |= (flags & EFLG_CF);
684 rc |= (flags & EFLG_ZF);
687 rc |= (flags & (EFLG_CF|EFLG_ZF));
690 rc |= (flags & EFLG_SF);
693 rc |= (flags & EFLG_PF);
696 rc |= (flags & EFLG_ZF);
699 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
703 /* Odd condition identifiers (lsb == 1) have inverted sense. */
704 return (!!rc ^ (condition & 1));
707 static void decode_register_operand(struct operand *op,
708 struct decode_cache *c,
711 unsigned reg = c->modrm_reg;
712 int highbyte_regs = c->rex_prefix == 0;
715 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
717 if ((c->d & ByteOp) && !inhibit_bytereg) {
718 op->ptr = decode_register(reg, c->regs, highbyte_regs);
719 op->val = *(u8 *)op->ptr;
722 op->ptr = decode_register(reg, c->regs, 0);
723 op->bytes = c->op_bytes;
726 op->val = *(u16 *)op->ptr;
729 op->val = *(u32 *)op->ptr;
732 op->val = *(u64 *) op->ptr;
736 op->orig_val = op->val;
739 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
740 struct x86_emulate_ops *ops)
742 struct decode_cache *c = &ctxt->decode;
744 int index_reg = 0, base_reg = 0, scale;
748 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
749 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
750 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
753 c->modrm = insn_fetch(u8, 1, c->eip);
754 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
755 c->modrm_reg |= (c->modrm & 0x38) >> 3;
756 c->modrm_rm |= (c->modrm & 0x07);
760 if (c->modrm_mod == 3) {
761 c->modrm_ptr = decode_register(c->modrm_rm,
762 c->regs, c->d & ByteOp);
763 c->modrm_val = *(unsigned long *)c->modrm_ptr;
767 if (c->ad_bytes == 2) {
768 unsigned bx = c->regs[VCPU_REGS_RBX];
769 unsigned bp = c->regs[VCPU_REGS_RBP];
770 unsigned si = c->regs[VCPU_REGS_RSI];
771 unsigned di = c->regs[VCPU_REGS_RDI];
773 /* 16-bit ModR/M decode. */
774 switch (c->modrm_mod) {
776 if (c->modrm_rm == 6)
777 c->modrm_ea += insn_fetch(u16, 2, c->eip);
780 c->modrm_ea += insn_fetch(s8, 1, c->eip);
783 c->modrm_ea += insn_fetch(u16, 2, c->eip);
786 switch (c->modrm_rm) {
788 c->modrm_ea += bx + si;
791 c->modrm_ea += bx + di;
794 c->modrm_ea += bp + si;
797 c->modrm_ea += bp + di;
806 if (c->modrm_mod != 0)
813 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
814 (c->modrm_rm == 6 && c->modrm_mod != 0))
815 if (!c->has_seg_override)
816 set_seg_override(c, VCPU_SREG_SS);
817 c->modrm_ea = (u16)c->modrm_ea;
819 /* 32/64-bit ModR/M decode. */
820 if ((c->modrm_rm & 7) == 4) {
821 sib = insn_fetch(u8, 1, c->eip);
822 index_reg |= (sib >> 3) & 7;
826 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
827 c->modrm_ea += insn_fetch(s32, 4, c->eip);
829 c->modrm_ea += c->regs[base_reg];
831 c->modrm_ea += c->regs[index_reg] << scale;
832 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
833 if (ctxt->mode == X86EMUL_MODE_PROT64)
836 c->modrm_ea += c->regs[c->modrm_rm];
837 switch (c->modrm_mod) {
839 if (c->modrm_rm == 5)
840 c->modrm_ea += insn_fetch(s32, 4, c->eip);
843 c->modrm_ea += insn_fetch(s8, 1, c->eip);
846 c->modrm_ea += insn_fetch(s32, 4, c->eip);
854 static int decode_abs(struct x86_emulate_ctxt *ctxt,
855 struct x86_emulate_ops *ops)
857 struct decode_cache *c = &ctxt->decode;
860 switch (c->ad_bytes) {
862 c->modrm_ea = insn_fetch(u16, 2, c->eip);
865 c->modrm_ea = insn_fetch(u32, 4, c->eip);
868 c->modrm_ea = insn_fetch(u64, 8, c->eip);
876 x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
878 struct decode_cache *c = &ctxt->decode;
880 int mode = ctxt->mode;
881 int def_op_bytes, def_ad_bytes, group;
883 /* Shadow copy of register state. Committed on successful emulation. */
885 memset(c, 0, sizeof(struct decode_cache));
886 c->eip = c->eip_orig = kvm_rip_read(ctxt->vcpu);
887 ctxt->cs_base = seg_base(ctxt, VCPU_SREG_CS);
888 memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs);
891 case X86EMUL_MODE_REAL:
892 case X86EMUL_MODE_PROT16:
893 def_op_bytes = def_ad_bytes = 2;
895 case X86EMUL_MODE_PROT32:
896 def_op_bytes = def_ad_bytes = 4;
899 case X86EMUL_MODE_PROT64:
908 c->op_bytes = def_op_bytes;
909 c->ad_bytes = def_ad_bytes;
911 /* Legacy prefixes. */
913 switch (c->b = insn_fetch(u8, 1, c->eip)) {
914 case 0x66: /* operand-size override */
915 /* switch between 2/4 bytes */
916 c->op_bytes = def_op_bytes ^ 6;
918 case 0x67: /* address-size override */
919 if (mode == X86EMUL_MODE_PROT64)
920 /* switch between 4/8 bytes */
921 c->ad_bytes = def_ad_bytes ^ 12;
923 /* switch between 2/4 bytes */
924 c->ad_bytes = def_ad_bytes ^ 6;
926 case 0x26: /* ES override */
927 case 0x2e: /* CS override */
928 case 0x36: /* SS override */
929 case 0x3e: /* DS override */
930 set_seg_override(c, (c->b >> 3) & 3);
932 case 0x64: /* FS override */
933 case 0x65: /* GS override */
934 set_seg_override(c, c->b & 7);
936 case 0x40 ... 0x4f: /* REX */
937 if (mode != X86EMUL_MODE_PROT64)
939 c->rex_prefix = c->b;
941 case 0xf0: /* LOCK */
944 case 0xf2: /* REPNE/REPNZ */
945 c->rep_prefix = REPNE_PREFIX;
947 case 0xf3: /* REP/REPE/REPZ */
948 c->rep_prefix = REPE_PREFIX;
954 /* Any legacy prefix after a REX prefix nullifies its effect. */
963 if (c->rex_prefix & 8)
964 c->op_bytes = 8; /* REX.W */
966 /* Opcode byte(s). */
967 c->d = opcode_table[c->b];
969 /* Two-byte opcode? */
972 c->b = insn_fetch(u8, 1, c->eip);
973 c->d = twobyte_table[c->b];
977 if (mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
978 kvm_report_emulation_failure(ctxt->vcpu, "invalid x86/64 instruction");;
983 group = c->d & GroupMask;
984 c->modrm = insn_fetch(u8, 1, c->eip);
987 group = (group << 3) + ((c->modrm >> 3) & 7);
988 if ((c->d & GroupDual) && (c->modrm >> 6) == 3)
989 c->d = group2_table[group];
991 c->d = group_table[group];
996 DPRINTF("Cannot emulate %02x\n", c->b);
1000 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
1003 /* ModRM and SIB bytes. */
1005 rc = decode_modrm(ctxt, ops);
1006 else if (c->d & MemAbs)
1007 rc = decode_abs(ctxt, ops);
1011 if (!c->has_seg_override)
1012 set_seg_override(c, VCPU_SREG_DS);
1014 if (!(!c->twobyte && c->b == 0x8d))
1015 c->modrm_ea += seg_override_base(ctxt, c);
1017 if (c->ad_bytes != 8)
1018 c->modrm_ea = (u32)c->modrm_ea;
1020 * Decode and fetch the source operand: register, memory
1023 switch (c->d & SrcMask) {
1027 decode_register_operand(&c->src, c, 0);
1036 c->src.bytes = (c->d & ByteOp) ? 1 :
1038 /* Don't fetch the address for invlpg: it could be unmapped. */
1039 if (c->twobyte && c->b == 0x01 && c->modrm_reg == 7)
1043 * For instructions with a ModR/M byte, switch to register
1044 * access if Mod = 3.
1046 if ((c->d & ModRM) && c->modrm_mod == 3) {
1047 c->src.type = OP_REG;
1048 c->src.val = c->modrm_val;
1049 c->src.ptr = c->modrm_ptr;
1052 c->src.type = OP_MEM;
1056 c->src.type = OP_IMM;
1057 c->src.ptr = (unsigned long *)c->eip;
1058 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1059 if (c->src.bytes == 8)
1061 /* NB. Immediates are sign-extended as necessary. */
1062 switch (c->src.bytes) {
1064 c->src.val = insn_fetch(s8, 1, c->eip);
1067 c->src.val = insn_fetch(s16, 2, c->eip);
1070 c->src.val = insn_fetch(s32, 4, c->eip);
1073 if ((c->d & SrcMask) == SrcImmU) {
1074 switch (c->src.bytes) {
1079 c->src.val &= 0xffff;
1082 c->src.val &= 0xffffffff;
1089 c->src.type = OP_IMM;
1090 c->src.ptr = (unsigned long *)c->eip;
1092 if ((c->d & SrcMask) == SrcImmByte)
1093 c->src.val = insn_fetch(s8, 1, c->eip);
1095 c->src.val = insn_fetch(u8, 1, c->eip);
1104 * Decode and fetch the second source operand: register, memory
1107 switch (c->d & Src2Mask) {
1112 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
1115 c->src2.type = OP_IMM;
1116 c->src2.ptr = (unsigned long *)c->eip;
1118 c->src2.val = insn_fetch(u8, 1, c->eip);
1121 c->src2.type = OP_IMM;
1122 c->src2.ptr = (unsigned long *)c->eip;
1124 c->src2.val = insn_fetch(u16, 2, c->eip);
1132 /* Decode and fetch the destination operand: register or memory. */
1133 switch (c->d & DstMask) {
1135 /* Special instructions do their own operand decoding. */
1138 decode_register_operand(&c->dst, c,
1139 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
1142 if ((c->d & ModRM) && c->modrm_mod == 3) {
1143 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1144 c->dst.type = OP_REG;
1145 c->dst.val = c->dst.orig_val = c->modrm_val;
1146 c->dst.ptr = c->modrm_ptr;
1149 c->dst.type = OP_MEM;
1152 c->dst.type = OP_REG;
1153 c->dst.bytes = c->op_bytes;
1154 c->dst.ptr = &c->regs[VCPU_REGS_RAX];
1155 switch (c->op_bytes) {
1157 c->dst.val = *(u8 *)c->dst.ptr;
1160 c->dst.val = *(u16 *)c->dst.ptr;
1163 c->dst.val = *(u32 *)c->dst.ptr;
1166 c->dst.orig_val = c->dst.val;
1170 if (c->rip_relative)
1171 c->modrm_ea += c->eip;
1174 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
1177 static inline void emulate_push(struct x86_emulate_ctxt *ctxt)
1179 struct decode_cache *c = &ctxt->decode;
1181 c->dst.type = OP_MEM;
1182 c->dst.bytes = c->op_bytes;
1183 c->dst.val = c->src.val;
1184 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1185 c->dst.ptr = (void *) register_address(c, ss_base(ctxt),
1186 c->regs[VCPU_REGS_RSP]);
1189 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1190 struct x86_emulate_ops *ops,
1191 void *dest, int len)
1193 struct decode_cache *c = &ctxt->decode;
1196 rc = ops->read_emulated(register_address(c, ss_base(ctxt),
1197 c->regs[VCPU_REGS_RSP]),
1198 dest, len, ctxt->vcpu);
1199 if (rc != X86EMUL_CONTINUE)
1202 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1206 static void emulate_push_sreg(struct x86_emulate_ctxt *ctxt, int seg)
1208 struct decode_cache *c = &ctxt->decode;
1209 struct kvm_segment segment;
1211 kvm_x86_ops->get_segment(ctxt->vcpu, &segment, seg);
1213 c->src.val = segment.selector;
1217 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1218 struct x86_emulate_ops *ops, int seg)
1220 struct decode_cache *c = &ctxt->decode;
1221 unsigned long selector;
1224 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1228 rc = kvm_load_segment_descriptor(ctxt->vcpu, (u16)selector, 1, seg);
1232 static void emulate_pusha(struct x86_emulate_ctxt *ctxt)
1234 struct decode_cache *c = &ctxt->decode;
1235 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1236 int reg = VCPU_REGS_RAX;
1238 while (reg <= VCPU_REGS_RDI) {
1239 (reg == VCPU_REGS_RSP) ?
1240 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1247 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1248 struct x86_emulate_ops *ops)
1250 struct decode_cache *c = &ctxt->decode;
1252 int reg = VCPU_REGS_RDI;
1254 while (reg >= VCPU_REGS_RAX) {
1255 if (reg == VCPU_REGS_RSP) {
1256 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1261 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1269 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1270 struct x86_emulate_ops *ops)
1272 struct decode_cache *c = &ctxt->decode;
1275 rc = emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1281 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1283 struct decode_cache *c = &ctxt->decode;
1284 switch (c->modrm_reg) {
1286 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1289 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1292 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1295 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1297 case 4: /* sal/shl */
1298 case 6: /* sal/shl */
1299 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1302 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1305 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1310 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1311 struct x86_emulate_ops *ops)
1313 struct decode_cache *c = &ctxt->decode;
1316 switch (c->modrm_reg) {
1317 case 0 ... 1: /* test */
1318 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1321 c->dst.val = ~c->dst.val;
1324 emulate_1op("neg", c->dst, ctxt->eflags);
1327 DPRINTF("Cannot emulate %02x\n", c->b);
1328 rc = X86EMUL_UNHANDLEABLE;
1334 static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
1335 struct x86_emulate_ops *ops)
1337 struct decode_cache *c = &ctxt->decode;
1339 switch (c->modrm_reg) {
1341 emulate_1op("inc", c->dst, ctxt->eflags);
1344 emulate_1op("dec", c->dst, ctxt->eflags);
1346 case 2: /* call near abs */ {
1349 c->eip = c->src.val;
1350 c->src.val = old_eip;
1354 case 4: /* jmp abs */
1355 c->eip = c->src.val;
1364 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1365 struct x86_emulate_ops *ops,
1366 unsigned long memop)
1368 struct decode_cache *c = &ctxt->decode;
1372 rc = ops->read_emulated(memop, &old, 8, ctxt->vcpu);
1373 if (rc != X86EMUL_CONTINUE)
1376 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1377 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1379 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1380 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1381 ctxt->eflags &= ~EFLG_ZF;
1384 new = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1385 (u32) c->regs[VCPU_REGS_RBX];
1387 rc = ops->cmpxchg_emulated(memop, &old, &new, 8, ctxt->vcpu);
1388 if (rc != X86EMUL_CONTINUE)
1390 ctxt->eflags |= EFLG_ZF;
1395 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1396 struct x86_emulate_ops *ops)
1398 struct decode_cache *c = &ctxt->decode;
1402 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1405 if (c->op_bytes == 4)
1406 c->eip = (u32)c->eip;
1407 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1410 rc = kvm_load_segment_descriptor(ctxt->vcpu, (u16)cs, 1, VCPU_SREG_CS);
1414 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1415 struct x86_emulate_ops *ops)
1418 struct decode_cache *c = &ctxt->decode;
1420 switch (c->dst.type) {
1422 /* The 4-byte case *is* correct:
1423 * in 64-bit mode we zero-extend.
1425 switch (c->dst.bytes) {
1427 *(u8 *)c->dst.ptr = (u8)c->dst.val;
1430 *(u16 *)c->dst.ptr = (u16)c->dst.val;
1433 *c->dst.ptr = (u32)c->dst.val;
1434 break; /* 64b: zero-ext */
1436 *c->dst.ptr = c->dst.val;
1442 rc = ops->cmpxchg_emulated(
1443 (unsigned long)c->dst.ptr,
1449 rc = ops->write_emulated(
1450 (unsigned long)c->dst.ptr,
1454 if (rc != X86EMUL_CONTINUE)
1466 static void toggle_interruptibility(struct x86_emulate_ctxt *ctxt, u32 mask)
1468 u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(ctxt->vcpu, mask);
1470 * an sti; sti; sequence only disable interrupts for the first
1471 * instruction. So, if the last instruction, be it emulated or
1472 * not, left the system with the INT_STI flag enabled, it
1473 * means that the last instruction is an sti. We should not
1474 * leave the flag on in this case. The same goes for mov ss
1476 if (!(int_shadow & mask))
1477 ctxt->interruptibility = mask;
1481 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1482 struct kvm_segment *cs, struct kvm_segment *ss)
1484 memset(cs, 0, sizeof(struct kvm_segment));
1485 kvm_x86_ops->get_segment(ctxt->vcpu, cs, VCPU_SREG_CS);
1486 memset(ss, 0, sizeof(struct kvm_segment));
1488 cs->l = 0; /* will be adjusted later */
1489 cs->base = 0; /* flat segment */
1490 cs->g = 1; /* 4kb granularity */
1491 cs->limit = 0xffffffff; /* 4GB limit */
1492 cs->type = 0x0b; /* Read, Execute, Accessed */
1494 cs->dpl = 0; /* will be adjusted later */
1499 ss->base = 0; /* flat segment */
1500 ss->limit = 0xffffffff; /* 4GB limit */
1501 ss->g = 1; /* 4kb granularity */
1503 ss->type = 0x03; /* Read/Write, Accessed */
1504 ss->db = 1; /* 32bit stack segment */
1510 emulate_syscall(struct x86_emulate_ctxt *ctxt)
1512 struct decode_cache *c = &ctxt->decode;
1513 struct kvm_segment cs, ss;
1516 /* syscall is not available in real mode */
1517 if (c->lock_prefix || ctxt->mode == X86EMUL_MODE_REAL
1518 || !kvm_read_cr0_bits(ctxt->vcpu, X86_CR0_PE))
1521 setup_syscalls_segments(ctxt, &cs, &ss);
1523 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1525 cs.selector = (u16)(msr_data & 0xfffc);
1526 ss.selector = (u16)(msr_data + 8);
1528 if (is_long_mode(ctxt->vcpu)) {
1532 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1533 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1535 c->regs[VCPU_REGS_RCX] = c->eip;
1536 if (is_long_mode(ctxt->vcpu)) {
1537 #ifdef CONFIG_X86_64
1538 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1540 kvm_x86_ops->get_msr(ctxt->vcpu,
1541 ctxt->mode == X86EMUL_MODE_PROT64 ?
1542 MSR_LSTAR : MSR_CSTAR, &msr_data);
1545 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
1546 ctxt->eflags &= ~(msr_data | EFLG_RF);
1550 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1551 c->eip = (u32)msr_data;
1553 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1560 emulate_sysenter(struct x86_emulate_ctxt *ctxt)
1562 struct decode_cache *c = &ctxt->decode;
1563 struct kvm_segment cs, ss;
1566 /* inject #UD if LOCK prefix is used */
1570 /* inject #GP if in real mode or paging is disabled */
1571 if (ctxt->mode == X86EMUL_MODE_REAL ||
1572 !kvm_read_cr0_bits(ctxt->vcpu, X86_CR0_PE)) {
1573 kvm_inject_gp(ctxt->vcpu, 0);
1577 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1578 * Therefore, we inject an #UD.
1580 if (ctxt->mode == X86EMUL_MODE_PROT64)
1583 setup_syscalls_segments(ctxt, &cs, &ss);
1585 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1586 switch (ctxt->mode) {
1587 case X86EMUL_MODE_PROT32:
1588 if ((msr_data & 0xfffc) == 0x0) {
1589 kvm_inject_gp(ctxt->vcpu, 0);
1593 case X86EMUL_MODE_PROT64:
1594 if (msr_data == 0x0) {
1595 kvm_inject_gp(ctxt->vcpu, 0);
1601 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1602 cs.selector = (u16)msr_data;
1603 cs.selector &= ~SELECTOR_RPL_MASK;
1604 ss.selector = cs.selector + 8;
1605 ss.selector &= ~SELECTOR_RPL_MASK;
1606 if (ctxt->mode == X86EMUL_MODE_PROT64
1607 || is_long_mode(ctxt->vcpu)) {
1612 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1613 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1615 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
1618 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
1619 c->regs[VCPU_REGS_RSP] = msr_data;
1625 emulate_sysexit(struct x86_emulate_ctxt *ctxt)
1627 struct decode_cache *c = &ctxt->decode;
1628 struct kvm_segment cs, ss;
1632 /* inject #UD if LOCK prefix is used */
1636 /* inject #GP if in real mode or paging is disabled */
1637 if (ctxt->mode == X86EMUL_MODE_REAL
1638 || !kvm_read_cr0_bits(ctxt->vcpu, X86_CR0_PE)) {
1639 kvm_inject_gp(ctxt->vcpu, 0);
1643 /* sysexit must be called from CPL 0 */
1644 if (kvm_x86_ops->get_cpl(ctxt->vcpu) != 0) {
1645 kvm_inject_gp(ctxt->vcpu, 0);
1649 setup_syscalls_segments(ctxt, &cs, &ss);
1651 if ((c->rex_prefix & 0x8) != 0x0)
1652 usermode = X86EMUL_MODE_PROT64;
1654 usermode = X86EMUL_MODE_PROT32;
1658 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1660 case X86EMUL_MODE_PROT32:
1661 cs.selector = (u16)(msr_data + 16);
1662 if ((msr_data & 0xfffc) == 0x0) {
1663 kvm_inject_gp(ctxt->vcpu, 0);
1666 ss.selector = (u16)(msr_data + 24);
1668 case X86EMUL_MODE_PROT64:
1669 cs.selector = (u16)(msr_data + 32);
1670 if (msr_data == 0x0) {
1671 kvm_inject_gp(ctxt->vcpu, 0);
1674 ss.selector = cs.selector + 8;
1679 cs.selector |= SELECTOR_RPL_MASK;
1680 ss.selector |= SELECTOR_RPL_MASK;
1682 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1683 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1685 c->eip = ctxt->vcpu->arch.regs[VCPU_REGS_RDX];
1686 c->regs[VCPU_REGS_RSP] = ctxt->vcpu->arch.regs[VCPU_REGS_RCX];
1692 x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1694 unsigned long memop = 0;
1696 unsigned long saved_eip = 0;
1697 struct decode_cache *c = &ctxt->decode;
1702 ctxt->interruptibility = 0;
1704 /* Shadow copy of register state. Committed on successful emulation.
1705 * NOTE: we can copy them from vcpu as x86_decode_insn() doesn't
1709 memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs);
1712 if (((c->d & ModRM) && (c->modrm_mod != 3)) || (c->d & MemAbs))
1713 memop = c->modrm_ea;
1715 if (c->rep_prefix && (c->d & String)) {
1716 /* All REP prefixes have the same first termination condition */
1717 if (c->regs[VCPU_REGS_RCX] == 0) {
1718 kvm_rip_write(ctxt->vcpu, c->eip);
1721 /* The second termination condition only applies for REPE
1722 * and REPNE. Test if the repeat string operation prefix is
1723 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
1724 * corresponding termination condition according to:
1725 * - if REPE/REPZ and ZF = 0 then done
1726 * - if REPNE/REPNZ and ZF = 1 then done
1728 if ((c->b == 0xa6) || (c->b == 0xa7) ||
1729 (c->b == 0xae) || (c->b == 0xaf)) {
1730 if ((c->rep_prefix == REPE_PREFIX) &&
1731 ((ctxt->eflags & EFLG_ZF) == 0)) {
1732 kvm_rip_write(ctxt->vcpu, c->eip);
1735 if ((c->rep_prefix == REPNE_PREFIX) &&
1736 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF)) {
1737 kvm_rip_write(ctxt->vcpu, c->eip);
1741 c->regs[VCPU_REGS_RCX]--;
1742 c->eip = kvm_rip_read(ctxt->vcpu);
1745 if (c->src.type == OP_MEM) {
1746 c->src.ptr = (unsigned long *)memop;
1748 rc = ops->read_emulated((unsigned long)c->src.ptr,
1752 if (rc != X86EMUL_CONTINUE)
1754 c->src.orig_val = c->src.val;
1757 if ((c->d & DstMask) == ImplicitOps)
1761 if (c->dst.type == OP_MEM) {
1762 c->dst.ptr = (unsigned long *)memop;
1763 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1766 unsigned long mask = ~(c->dst.bytes * 8 - 1);
1768 c->dst.ptr = (void *)c->dst.ptr +
1769 (c->src.val & mask) / 8;
1771 if (!(c->d & Mov)) {
1772 /* optimisation - avoid slow emulated read */
1773 rc = ops->read_emulated((unsigned long)c->dst.ptr,
1777 if (rc != X86EMUL_CONTINUE)
1781 c->dst.orig_val = c->dst.val;
1791 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
1793 case 0x06: /* push es */
1794 emulate_push_sreg(ctxt, VCPU_SREG_ES);
1796 case 0x07: /* pop es */
1797 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
1803 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
1805 case 0x0e: /* push cs */
1806 emulate_push_sreg(ctxt, VCPU_SREG_CS);
1810 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
1812 case 0x16: /* push ss */
1813 emulate_push_sreg(ctxt, VCPU_SREG_SS);
1815 case 0x17: /* pop ss */
1816 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
1822 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
1824 case 0x1e: /* push ds */
1825 emulate_push_sreg(ctxt, VCPU_SREG_DS);
1827 case 0x1f: /* pop ds */
1828 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
1834 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
1838 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
1842 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
1846 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
1848 case 0x40 ... 0x47: /* inc r16/r32 */
1849 emulate_1op("inc", c->dst, ctxt->eflags);
1851 case 0x48 ... 0x4f: /* dec r16/r32 */
1852 emulate_1op("dec", c->dst, ctxt->eflags);
1854 case 0x50 ... 0x57: /* push reg */
1857 case 0x58 ... 0x5f: /* pop reg */
1859 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
1863 case 0x60: /* pusha */
1864 emulate_pusha(ctxt);
1866 case 0x61: /* popa */
1867 rc = emulate_popa(ctxt, ops);
1871 case 0x63: /* movsxd */
1872 if (ctxt->mode != X86EMUL_MODE_PROT64)
1873 goto cannot_emulate;
1874 c->dst.val = (s32) c->src.val;
1876 case 0x68: /* push imm */
1877 case 0x6a: /* push imm8 */
1880 case 0x6c: /* insb */
1881 case 0x6d: /* insw/insd */
1882 if (kvm_emulate_pio_string(ctxt->vcpu,
1884 (c->d & ByteOp) ? 1 : c->op_bytes,
1886 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1,
1887 (ctxt->eflags & EFLG_DF),
1888 register_address(c, es_base(ctxt),
1889 c->regs[VCPU_REGS_RDI]),
1891 c->regs[VCPU_REGS_RDX]) == 0) {
1896 case 0x6e: /* outsb */
1897 case 0x6f: /* outsw/outsd */
1898 if (kvm_emulate_pio_string(ctxt->vcpu,
1900 (c->d & ByteOp) ? 1 : c->op_bytes,
1902 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1,
1903 (ctxt->eflags & EFLG_DF),
1905 seg_override_base(ctxt, c),
1906 c->regs[VCPU_REGS_RSI]),
1908 c->regs[VCPU_REGS_RDX]) == 0) {
1913 case 0x70 ... 0x7f: /* jcc (short) */
1914 if (test_cc(c->b, ctxt->eflags))
1915 jmp_rel(c, c->src.val);
1917 case 0x80 ... 0x83: /* Grp1 */
1918 switch (c->modrm_reg) {
1938 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1940 case 0x86 ... 0x87: /* xchg */
1942 /* Write back the register source. */
1943 switch (c->dst.bytes) {
1945 *(u8 *) c->src.ptr = (u8) c->dst.val;
1948 *(u16 *) c->src.ptr = (u16) c->dst.val;
1951 *c->src.ptr = (u32) c->dst.val;
1952 break; /* 64b reg: zero-extend */
1954 *c->src.ptr = c->dst.val;
1958 * Write back the memory destination with implicit LOCK
1961 c->dst.val = c->src.val;
1964 case 0x88 ... 0x8b: /* mov */
1966 case 0x8c: { /* mov r/m, sreg */
1967 struct kvm_segment segreg;
1969 if (c->modrm_reg <= 5)
1970 kvm_get_segment(ctxt->vcpu, &segreg, c->modrm_reg);
1972 printk(KERN_INFO "0x8c: Invalid segreg in modrm byte 0x%02x\n",
1974 goto cannot_emulate;
1976 c->dst.val = segreg.selector;
1979 case 0x8d: /* lea r16/r32, m */
1980 c->dst.val = c->modrm_ea;
1982 case 0x8e: { /* mov seg, r/m16 */
1988 if (c->modrm_reg == VCPU_SREG_SS)
1989 toggle_interruptibility(ctxt, X86_SHADOW_INT_MOV_SS);
1991 if (c->modrm_reg <= 5) {
1992 type_bits = (c->modrm_reg == 1) ? 9 : 1;
1993 err = kvm_load_segment_descriptor(ctxt->vcpu, sel,
1994 type_bits, c->modrm_reg);
1996 printk(KERN_INFO "Invalid segreg in modrm byte 0x%02x\n",
1998 goto cannot_emulate;
2002 goto cannot_emulate;
2004 c->dst.type = OP_NONE; /* Disable writeback. */
2007 case 0x8f: /* pop (sole member of Grp1a) */
2008 rc = emulate_grp1a(ctxt, ops);
2012 case 0x90: /* nop / xchg r8,rax */
2013 if (!(c->rex_prefix & 1)) { /* nop */
2014 c->dst.type = OP_NONE;
2017 case 0x91 ... 0x97: /* xchg reg,rax */
2018 c->src.type = c->dst.type = OP_REG;
2019 c->src.bytes = c->dst.bytes = c->op_bytes;
2020 c->src.ptr = (unsigned long *) &c->regs[VCPU_REGS_RAX];
2021 c->src.val = *(c->src.ptr);
2023 case 0x9c: /* pushf */
2024 c->src.val = (unsigned long) ctxt->eflags;
2027 case 0x9d: /* popf */
2028 c->dst.type = OP_REG;
2029 c->dst.ptr = (unsigned long *) &ctxt->eflags;
2030 c->dst.bytes = c->op_bytes;
2031 goto pop_instruction;
2032 case 0xa0 ... 0xa1: /* mov */
2033 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2034 c->dst.val = c->src.val;
2036 case 0xa2 ... 0xa3: /* mov */
2037 c->dst.val = (unsigned long)c->regs[VCPU_REGS_RAX];
2039 case 0xa4 ... 0xa5: /* movs */
2040 c->dst.type = OP_MEM;
2041 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2042 c->dst.ptr = (unsigned long *)register_address(c,
2044 c->regs[VCPU_REGS_RDI]);
2045 rc = ops->read_emulated(register_address(c,
2046 seg_override_base(ctxt, c),
2047 c->regs[VCPU_REGS_RSI]),
2049 c->dst.bytes, ctxt->vcpu);
2050 if (rc != X86EMUL_CONTINUE)
2052 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2053 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2055 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2056 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2059 case 0xa6 ... 0xa7: /* cmps */
2060 c->src.type = OP_NONE; /* Disable writeback. */
2061 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2062 c->src.ptr = (unsigned long *)register_address(c,
2063 seg_override_base(ctxt, c),
2064 c->regs[VCPU_REGS_RSI]);
2065 rc = ops->read_emulated((unsigned long)c->src.ptr,
2069 if (rc != X86EMUL_CONTINUE)
2072 c->dst.type = OP_NONE; /* Disable writeback. */
2073 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2074 c->dst.ptr = (unsigned long *)register_address(c,
2076 c->regs[VCPU_REGS_RDI]);
2077 rc = ops->read_emulated((unsigned long)c->dst.ptr,
2081 if (rc != X86EMUL_CONTINUE)
2084 DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.ptr, c->dst.ptr);
2086 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2088 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2089 (ctxt->eflags & EFLG_DF) ? -c->src.bytes
2091 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2092 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2096 case 0xaa ... 0xab: /* stos */
2097 c->dst.type = OP_MEM;
2098 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2099 c->dst.ptr = (unsigned long *)register_address(c,
2101 c->regs[VCPU_REGS_RDI]);
2102 c->dst.val = c->regs[VCPU_REGS_RAX];
2103 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2104 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2107 case 0xac ... 0xad: /* lods */
2108 c->dst.type = OP_REG;
2109 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2110 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2111 rc = ops->read_emulated(register_address(c,
2112 seg_override_base(ctxt, c),
2113 c->regs[VCPU_REGS_RSI]),
2117 if (rc != X86EMUL_CONTINUE)
2119 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2120 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2123 case 0xae ... 0xaf: /* scas */
2124 DPRINTF("Urk! I don't handle SCAS.\n");
2125 goto cannot_emulate;
2126 case 0xb0 ... 0xbf: /* mov r, imm */
2131 case 0xc3: /* ret */
2132 c->dst.type = OP_REG;
2133 c->dst.ptr = &c->eip;
2134 c->dst.bytes = c->op_bytes;
2135 goto pop_instruction;
2136 case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */
2138 c->dst.val = c->src.val;
2140 case 0xcb: /* ret far */
2141 rc = emulate_ret_far(ctxt, ops);
2145 case 0xd0 ... 0xd1: /* Grp2 */
2149 case 0xd2 ... 0xd3: /* Grp2 */
2150 c->src.val = c->regs[VCPU_REGS_RCX];
2153 case 0xe4: /* inb */
2158 case 0xe6: /* outb */
2159 case 0xe7: /* out */
2163 case 0xe8: /* call (near) */ {
2164 long int rel = c->src.val;
2165 c->src.val = (unsigned long) c->eip;
2170 case 0xe9: /* jmp rel */
2172 case 0xea: /* jmp far */
2173 if (kvm_load_segment_descriptor(ctxt->vcpu, c->src2.val, 9,
2174 VCPU_SREG_CS) < 0) {
2175 DPRINTF("jmp far: Failed to load CS descriptor\n");
2176 goto cannot_emulate;
2179 c->eip = c->src.val;
2182 jmp: /* jmp rel short */
2183 jmp_rel(c, c->src.val);
2184 c->dst.type = OP_NONE; /* Disable writeback. */
2186 case 0xec: /* in al,dx */
2187 case 0xed: /* in (e/r)ax,dx */
2188 port = c->regs[VCPU_REGS_RDX];
2191 case 0xee: /* out al,dx */
2192 case 0xef: /* out (e/r)ax,dx */
2193 port = c->regs[VCPU_REGS_RDX];
2195 do_io: if (kvm_emulate_pio(ctxt->vcpu, io_dir_in,
2196 (c->d & ByteOp) ? 1 : c->op_bytes,
2199 goto cannot_emulate;
2202 case 0xf4: /* hlt */
2203 ctxt->vcpu->arch.halt_request = 1;
2205 case 0xf5: /* cmc */
2206 /* complement carry flag from eflags reg */
2207 ctxt->eflags ^= EFLG_CF;
2208 c->dst.type = OP_NONE; /* Disable writeback. */
2210 case 0xf6 ... 0xf7: /* Grp3 */
2211 rc = emulate_grp3(ctxt, ops);
2215 case 0xf8: /* clc */
2216 ctxt->eflags &= ~EFLG_CF;
2217 c->dst.type = OP_NONE; /* Disable writeback. */
2219 case 0xfa: /* cli */
2220 ctxt->eflags &= ~X86_EFLAGS_IF;
2221 c->dst.type = OP_NONE; /* Disable writeback. */
2223 case 0xfb: /* sti */
2224 toggle_interruptibility(ctxt, X86_SHADOW_INT_STI);
2225 ctxt->eflags |= X86_EFLAGS_IF;
2226 c->dst.type = OP_NONE; /* Disable writeback. */
2228 case 0xfc: /* cld */
2229 ctxt->eflags &= ~EFLG_DF;
2230 c->dst.type = OP_NONE; /* Disable writeback. */
2232 case 0xfd: /* std */
2233 ctxt->eflags |= EFLG_DF;
2234 c->dst.type = OP_NONE; /* Disable writeback. */
2236 case 0xfe ... 0xff: /* Grp4/Grp5 */
2237 rc = emulate_grp45(ctxt, ops);
2244 rc = writeback(ctxt, ops);
2248 /* Commit shadow register state. */
2249 memcpy(ctxt->vcpu->arch.regs, c->regs, sizeof c->regs);
2250 kvm_rip_write(ctxt->vcpu, c->eip);
2253 if (rc == X86EMUL_UNHANDLEABLE) {
2261 case 0x01: /* lgdt, lidt, lmsw */
2262 switch (c->modrm_reg) {
2264 unsigned long address;
2266 case 0: /* vmcall */
2267 if (c->modrm_mod != 3 || c->modrm_rm != 1)
2268 goto cannot_emulate;
2270 rc = kvm_fix_hypercall(ctxt->vcpu);
2274 /* Let the processor re-execute the fixed hypercall */
2275 c->eip = kvm_rip_read(ctxt->vcpu);
2276 /* Disable writeback. */
2277 c->dst.type = OP_NONE;
2280 rc = read_descriptor(ctxt, ops, c->src.ptr,
2281 &size, &address, c->op_bytes);
2284 realmode_lgdt(ctxt->vcpu, size, address);
2285 /* Disable writeback. */
2286 c->dst.type = OP_NONE;
2288 case 3: /* lidt/vmmcall */
2289 if (c->modrm_mod == 3) {
2290 switch (c->modrm_rm) {
2292 rc = kvm_fix_hypercall(ctxt->vcpu);
2297 goto cannot_emulate;
2300 rc = read_descriptor(ctxt, ops, c->src.ptr,
2305 realmode_lidt(ctxt->vcpu, size, address);
2307 /* Disable writeback. */
2308 c->dst.type = OP_NONE;
2312 c->dst.val = realmode_get_cr(ctxt->vcpu, 0);
2315 realmode_lmsw(ctxt->vcpu, (u16)c->src.val,
2317 c->dst.type = OP_NONE;
2320 emulate_invlpg(ctxt->vcpu, memop);
2321 /* Disable writeback. */
2322 c->dst.type = OP_NONE;
2325 goto cannot_emulate;
2328 case 0x05: /* syscall */
2329 if (emulate_syscall(ctxt) == -1)
2330 goto cannot_emulate;
2335 emulate_clts(ctxt->vcpu);
2336 c->dst.type = OP_NONE;
2338 case 0x08: /* invd */
2339 case 0x09: /* wbinvd */
2340 case 0x0d: /* GrpP (prefetch) */
2341 case 0x18: /* Grp16 (prefetch/nop) */
2342 c->dst.type = OP_NONE;
2344 case 0x20: /* mov cr, reg */
2345 if (c->modrm_mod != 3)
2346 goto cannot_emulate;
2347 c->regs[c->modrm_rm] =
2348 realmode_get_cr(ctxt->vcpu, c->modrm_reg);
2349 c->dst.type = OP_NONE; /* no writeback */
2351 case 0x21: /* mov from dr to reg */
2352 if (c->modrm_mod != 3)
2353 goto cannot_emulate;
2354 rc = emulator_get_dr(ctxt, c->modrm_reg, &c->regs[c->modrm_rm]);
2356 goto cannot_emulate;
2357 c->dst.type = OP_NONE; /* no writeback */
2359 case 0x22: /* mov reg, cr */
2360 if (c->modrm_mod != 3)
2361 goto cannot_emulate;
2362 realmode_set_cr(ctxt->vcpu,
2363 c->modrm_reg, c->modrm_val, &ctxt->eflags);
2364 c->dst.type = OP_NONE;
2366 case 0x23: /* mov from reg to dr */
2367 if (c->modrm_mod != 3)
2368 goto cannot_emulate;
2369 rc = emulator_set_dr(ctxt, c->modrm_reg,
2370 c->regs[c->modrm_rm]);
2372 goto cannot_emulate;
2373 c->dst.type = OP_NONE; /* no writeback */
2377 msr_data = (u32)c->regs[VCPU_REGS_RAX]
2378 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
2379 rc = kvm_set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data);
2381 kvm_inject_gp(ctxt->vcpu, 0);
2382 c->eip = kvm_rip_read(ctxt->vcpu);
2384 rc = X86EMUL_CONTINUE;
2385 c->dst.type = OP_NONE;
2389 rc = kvm_get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data);
2391 kvm_inject_gp(ctxt->vcpu, 0);
2392 c->eip = kvm_rip_read(ctxt->vcpu);
2394 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
2395 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
2397 rc = X86EMUL_CONTINUE;
2398 c->dst.type = OP_NONE;
2400 case 0x34: /* sysenter */
2401 if (emulate_sysenter(ctxt) == -1)
2402 goto cannot_emulate;
2406 case 0x35: /* sysexit */
2407 if (emulate_sysexit(ctxt) == -1)
2408 goto cannot_emulate;
2412 case 0x40 ... 0x4f: /* cmov */
2413 c->dst.val = c->dst.orig_val = c->src.val;
2414 if (!test_cc(c->b, ctxt->eflags))
2415 c->dst.type = OP_NONE; /* no writeback */
2417 case 0x80 ... 0x8f: /* jnz rel, etc*/
2418 if (test_cc(c->b, ctxt->eflags))
2419 jmp_rel(c, c->src.val);
2420 c->dst.type = OP_NONE;
2422 case 0xa0: /* push fs */
2423 emulate_push_sreg(ctxt, VCPU_SREG_FS);
2425 case 0xa1: /* pop fs */
2426 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
2432 c->dst.type = OP_NONE;
2433 /* only subword offset */
2434 c->src.val &= (c->dst.bytes << 3) - 1;
2435 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
2437 case 0xa4: /* shld imm8, r, r/m */
2438 case 0xa5: /* shld cl, r, r/m */
2439 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
2441 case 0xa8: /* push gs */
2442 emulate_push_sreg(ctxt, VCPU_SREG_GS);
2444 case 0xa9: /* pop gs */
2445 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
2451 /* only subword offset */
2452 c->src.val &= (c->dst.bytes << 3) - 1;
2453 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
2455 case 0xac: /* shrd imm8, r, r/m */
2456 case 0xad: /* shrd cl, r, r/m */
2457 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
2459 case 0xae: /* clflush */
2461 case 0xb0 ... 0xb1: /* cmpxchg */
2463 * Save real source value, then compare EAX against
2466 c->src.orig_val = c->src.val;
2467 c->src.val = c->regs[VCPU_REGS_RAX];
2468 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2469 if (ctxt->eflags & EFLG_ZF) {
2470 /* Success: write back to memory. */
2471 c->dst.val = c->src.orig_val;
2473 /* Failure: write the value we saw to EAX. */
2474 c->dst.type = OP_REG;
2475 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2480 /* only subword offset */
2481 c->src.val &= (c->dst.bytes << 3) - 1;
2482 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
2484 case 0xb6 ... 0xb7: /* movzx */
2485 c->dst.bytes = c->op_bytes;
2486 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
2489 case 0xba: /* Grp8 */
2490 switch (c->modrm_reg & 3) {
2503 /* only subword offset */
2504 c->src.val &= (c->dst.bytes << 3) - 1;
2505 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
2507 case 0xbe ... 0xbf: /* movsx */
2508 c->dst.bytes = c->op_bytes;
2509 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
2512 case 0xc3: /* movnti */
2513 c->dst.bytes = c->op_bytes;
2514 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
2517 case 0xc7: /* Grp9 (cmpxchg8b) */
2518 rc = emulate_grp9(ctxt, ops, memop);
2521 c->dst.type = OP_NONE;
2527 DPRINTF("Cannot emulate %02x\n", c->b);