1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affilates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
26 #include <public/xen.h>
27 #define DPRINTF(_f, _a ...) printf(_f , ## _a)
29 #include <linux/kvm_host.h>
30 #include "kvm_cache_regs.h"
31 #define DPRINTF(x...) do {} while (0)
33 #include <linux/module.h>
34 #include <asm/kvm_emulate.h>
40 * Opcode effective-address decode tables.
41 * Note that we only emulate instructions that have at least one memory
42 * operand (excluding implicit stack references). We assume that stack
43 * references and instruction fetches will never occur in special memory
44 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
48 /* Operand sizes: 8-bit operands or specified/overridden size. */
49 #define ByteOp (1<<0) /* 8-bit operands. */
50 /* Destination operand type. */
51 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
52 #define DstReg (2<<1) /* Register operand. */
53 #define DstMem (3<<1) /* Memory operand. */
54 #define DstAcc (4<<1) /* Destination Accumulator */
55 #define DstDI (5<<1) /* Destination is in ES:(E)DI */
56 #define DstMem64 (6<<1) /* 64bit memory operand */
57 #define DstMask (7<<1)
58 /* Source operand type. */
59 #define SrcNone (0<<4) /* No source operand. */
60 #define SrcImplicit (0<<4) /* Source operand is implicit in the opcode. */
61 #define SrcReg (1<<4) /* Register operand. */
62 #define SrcMem (2<<4) /* Memory operand. */
63 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
64 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
65 #define SrcImm (5<<4) /* Immediate operand. */
66 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
67 #define SrcOne (7<<4) /* Implied '1' */
68 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
69 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
70 #define SrcSI (0xa<<4) /* Source is in the DS:RSI */
71 #define SrcImmFAddr (0xb<<4) /* Source is immediate far address */
72 #define SrcMemFAddr (0xc<<4) /* Source is far address in memory */
73 #define SrcMask (0xf<<4)
74 /* Generic ModRM decode. */
76 /* Destination is only written; never read. */
79 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
80 #define String (1<<12) /* String instruction (rep capable) */
81 #define Stack (1<<13) /* Stack instruction (push/pop) */
82 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
83 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
84 #define GroupMask 0xff /* Group number stored in bits 0:7 */
86 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
87 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
89 /* Source 2 operand type */
90 #define Src2None (0<<29)
91 #define Src2CL (1<<29)
92 #define Src2ImmByte (2<<29)
93 #define Src2One (3<<29)
94 #define Src2Mask (7<<29)
97 Group1_80, Group1_81, Group1_82, Group1_83,
98 Group1A, Group3_Byte, Group3, Group4, Group5, Group7,
102 static u32 opcode_table[256] = {
104 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
105 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
106 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
107 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
109 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
110 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
111 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
112 ImplicitOps | Stack | No64, 0,
114 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
115 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
116 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
117 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
119 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
120 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
121 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
122 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
124 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
125 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
126 DstAcc | SrcImmByte, DstAcc | SrcImm, 0, 0,
128 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
129 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
130 ByteOp | DstAcc | SrcImmByte, DstAcc | SrcImm, 0, 0,
132 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
133 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
134 ByteOp | DstAcc | SrcImmByte, DstAcc | SrcImm, 0, 0,
136 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
137 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
138 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
141 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
143 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
145 SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack,
146 SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack,
148 DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack,
149 DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack,
151 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
152 0, DstReg | SrcMem32 | ModRM | Mov /* movsxd (x86/64) */ ,
155 SrcImm | Mov | Stack, 0, SrcImmByte | Mov | Stack, 0,
156 DstDI | ByteOp | Mov | String, DstDI | Mov | String, /* insb, insw/insd */
157 SrcSI | ByteOp | ImplicitOps | String, SrcSI | ImplicitOps | String, /* outsb, outsw/outsd */
159 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
160 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
162 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
163 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
165 Group | Group1_80, Group | Group1_81,
166 Group | Group1_82, Group | Group1_83,
167 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
168 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
170 ByteOp | DstMem | SrcReg | ModRM | Mov, DstMem | SrcReg | ModRM | Mov,
171 ByteOp | DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
172 DstMem | SrcReg | ModRM | Mov, ModRM | DstReg,
173 ImplicitOps | SrcMem | ModRM, Group | Group1A,
175 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
177 0, 0, SrcImmFAddr | No64, 0,
178 ImplicitOps | Stack, ImplicitOps | Stack, 0, 0,
180 ByteOp | DstReg | SrcMem | Mov | MemAbs, DstReg | SrcMem | Mov | MemAbs,
181 ByteOp | DstMem | SrcReg | Mov | MemAbs, DstMem | SrcReg | Mov | MemAbs,
182 ByteOp | SrcSI | DstDI | Mov | String, SrcSI | DstDI | Mov | String,
183 ByteOp | SrcSI | DstDI | String, SrcSI | DstDI | String,
185 DstAcc | SrcImmByte | ByteOp, DstAcc | SrcImm, ByteOp | DstDI | Mov | String, DstDI | Mov | String,
186 ByteOp | SrcSI | DstAcc | Mov | String, SrcSI | DstAcc | Mov | String,
187 ByteOp | DstDI | String, DstDI | String,
189 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
190 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
191 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
192 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
194 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
195 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
196 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
197 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
199 ByteOp | DstMem | SrcImm | ModRM, DstMem | SrcImmByte | ModRM,
200 0, ImplicitOps | Stack, 0, 0,
201 ByteOp | DstMem | SrcImm | ModRM | Mov, DstMem | SrcImm | ModRM | Mov,
203 0, 0, 0, ImplicitOps | Stack,
204 ImplicitOps, SrcImmByte, ImplicitOps | No64, ImplicitOps,
206 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
207 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
210 0, 0, 0, 0, 0, 0, 0, 0,
213 ByteOp | SrcImmUByte | DstAcc, SrcImmUByte | DstAcc,
214 ByteOp | SrcImmUByte | DstAcc, SrcImmUByte | DstAcc,
216 SrcImm | Stack, SrcImm | ImplicitOps,
217 SrcImmFAddr | No64, SrcImmByte | ImplicitOps,
218 SrcNone | ByteOp | DstAcc, SrcNone | DstAcc,
219 SrcNone | ByteOp | DstAcc, SrcNone | DstAcc,
222 ImplicitOps | Priv, ImplicitOps, Group | Group3_Byte, Group | Group3,
224 ImplicitOps, 0, ImplicitOps, ImplicitOps,
225 ImplicitOps, ImplicitOps, Group | Group4, Group | Group5,
228 static u32 twobyte_table[256] = {
230 0, Group | GroupDual | Group7, 0, 0,
231 0, ImplicitOps, ImplicitOps | Priv, 0,
232 ImplicitOps | Priv, ImplicitOps | Priv, 0, 0,
233 0, ImplicitOps | ModRM, 0, 0,
235 0, 0, 0, 0, 0, 0, 0, 0, ImplicitOps | ModRM, 0, 0, 0, 0, 0, 0, 0,
237 ModRM | ImplicitOps | Priv, ModRM | Priv,
238 ModRM | ImplicitOps | Priv, ModRM | Priv,
240 0, 0, 0, 0, 0, 0, 0, 0,
242 ImplicitOps | Priv, 0, ImplicitOps | Priv, 0,
243 ImplicitOps, ImplicitOps | Priv, 0, 0,
244 0, 0, 0, 0, 0, 0, 0, 0,
246 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
247 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
248 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
249 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
251 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
252 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
253 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
254 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
256 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
258 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
260 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
262 SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm,
263 SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm,
265 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
267 ImplicitOps | Stack, ImplicitOps | Stack,
268 0, DstMem | SrcReg | ModRM | BitOp,
269 DstMem | SrcReg | Src2ImmByte | ModRM,
270 DstMem | SrcReg | Src2CL | ModRM, 0, 0,
272 ImplicitOps | Stack, ImplicitOps | Stack,
273 0, DstMem | SrcReg | ModRM | BitOp | Lock,
274 DstMem | SrcReg | Src2ImmByte | ModRM,
275 DstMem | SrcReg | Src2CL | ModRM,
278 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
279 0, DstMem | SrcReg | ModRM | BitOp | Lock,
280 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
281 DstReg | SrcMem16 | ModRM | Mov,
284 Group | Group8, DstMem | SrcReg | ModRM | BitOp | Lock,
285 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
286 DstReg | SrcMem16 | ModRM | Mov,
288 0, 0, 0, DstMem | SrcReg | ModRM | Mov,
289 0, 0, 0, Group | GroupDual | Group9,
290 0, 0, 0, 0, 0, 0, 0, 0,
292 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
294 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
296 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
299 static u32 group_table[] = {
301 ByteOp | DstMem | SrcImm | ModRM | Lock,
302 ByteOp | DstMem | SrcImm | ModRM | Lock,
303 ByteOp | DstMem | SrcImm | ModRM | Lock,
304 ByteOp | DstMem | SrcImm | ModRM | Lock,
305 ByteOp | DstMem | SrcImm | ModRM | Lock,
306 ByteOp | DstMem | SrcImm | ModRM | Lock,
307 ByteOp | DstMem | SrcImm | ModRM | Lock,
308 ByteOp | DstMem | SrcImm | ModRM,
310 DstMem | SrcImm | ModRM | Lock,
311 DstMem | SrcImm | ModRM | Lock,
312 DstMem | SrcImm | ModRM | Lock,
313 DstMem | SrcImm | ModRM | Lock,
314 DstMem | SrcImm | ModRM | Lock,
315 DstMem | SrcImm | ModRM | Lock,
316 DstMem | SrcImm | ModRM | Lock,
317 DstMem | SrcImm | ModRM,
319 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
320 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
321 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
322 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
323 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
324 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
325 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
326 ByteOp | DstMem | SrcImm | ModRM | No64,
328 DstMem | SrcImmByte | ModRM | Lock,
329 DstMem | SrcImmByte | ModRM | Lock,
330 DstMem | SrcImmByte | ModRM | Lock,
331 DstMem | SrcImmByte | ModRM | Lock,
332 DstMem | SrcImmByte | ModRM | Lock,
333 DstMem | SrcImmByte | ModRM | Lock,
334 DstMem | SrcImmByte | ModRM | Lock,
335 DstMem | SrcImmByte | ModRM,
337 DstMem | SrcNone | ModRM | Mov | Stack, 0, 0, 0, 0, 0, 0, 0,
339 ByteOp | SrcImm | DstMem | ModRM, 0,
340 ByteOp | DstMem | SrcNone | ModRM, ByteOp | DstMem | SrcNone | ModRM,
343 DstMem | SrcImm | ModRM, 0,
344 DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM,
347 ByteOp | DstMem | SrcNone | ModRM, ByteOp | DstMem | SrcNone | ModRM,
350 DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM,
351 SrcMem | ModRM | Stack, 0,
352 SrcMem | ModRM | Stack, SrcMemFAddr | ModRM | ImplicitOps,
353 SrcMem | ModRM | Stack, 0,
355 0, 0, ModRM | SrcMem | Priv, ModRM | SrcMem | Priv,
356 SrcNone | ModRM | DstMem | Mov, 0,
357 SrcMem16 | ModRM | Mov | Priv, SrcMem | ModRM | ByteOp | Priv,
360 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM | Lock,
361 DstMem | SrcImmByte | ModRM | Lock, DstMem | SrcImmByte | ModRM | Lock,
363 0, DstMem64 | ModRM | Lock, 0, 0, 0, 0, 0, 0,
366 static u32 group2_table[] = {
368 SrcNone | ModRM | Priv, 0, 0, SrcNone | ModRM | Priv,
369 SrcNone | ModRM | DstMem | Mov, 0,
370 SrcMem16 | ModRM | Mov | Priv, 0,
372 0, 0, 0, 0, 0, 0, 0, 0,
375 /* EFLAGS bit definitions. */
376 #define EFLG_ID (1<<21)
377 #define EFLG_VIP (1<<20)
378 #define EFLG_VIF (1<<19)
379 #define EFLG_AC (1<<18)
380 #define EFLG_VM (1<<17)
381 #define EFLG_RF (1<<16)
382 #define EFLG_IOPL (3<<12)
383 #define EFLG_NT (1<<14)
384 #define EFLG_OF (1<<11)
385 #define EFLG_DF (1<<10)
386 #define EFLG_IF (1<<9)
387 #define EFLG_TF (1<<8)
388 #define EFLG_SF (1<<7)
389 #define EFLG_ZF (1<<6)
390 #define EFLG_AF (1<<4)
391 #define EFLG_PF (1<<2)
392 #define EFLG_CF (1<<0)
395 * Instruction emulation:
396 * Most instructions are emulated directly via a fragment of inline assembly
397 * code. This allows us to save/restore EFLAGS and thus very easily pick up
398 * any modified flags.
401 #if defined(CONFIG_X86_64)
402 #define _LO32 "k" /* force 32-bit operand */
403 #define _STK "%%rsp" /* stack pointer */
404 #elif defined(__i386__)
405 #define _LO32 "" /* force 32-bit operand */
406 #define _STK "%%esp" /* stack pointer */
410 * These EFLAGS bits are restored from saved value during emulation, and
411 * any changes are written back to the saved value after emulation.
413 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
415 /* Before executing instruction: restore necessary bits in EFLAGS. */
416 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
417 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
418 "movl %"_sav",%"_LO32 _tmp"; " \
421 "movl %"_msk",%"_LO32 _tmp"; " \
422 "andl %"_LO32 _tmp",("_STK"); " \
424 "notl %"_LO32 _tmp"; " \
425 "andl %"_LO32 _tmp",("_STK"); " \
426 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
428 "orl %"_LO32 _tmp",("_STK"); " \
432 /* After executing instruction: write-back necessary bits in EFLAGS. */
433 #define _POST_EFLAGS(_sav, _msk, _tmp) \
434 /* _sav |= EFLAGS & _msk; */ \
437 "andl %"_msk",%"_LO32 _tmp"; " \
438 "orl %"_LO32 _tmp",%"_sav"; "
446 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix) \
448 __asm__ __volatile__ ( \
449 _PRE_EFLAGS("0", "4", "2") \
450 _op _suffix " %"_x"3,%1; " \
451 _POST_EFLAGS("0", "4", "2") \
452 : "=m" (_eflags), "=m" ((_dst).val), \
454 : _y ((_src).val), "i" (EFLAGS_MASK)); \
458 /* Raw emulation: instruction has two explicit operands. */
459 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
461 unsigned long _tmp; \
463 switch ((_dst).bytes) { \
465 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w"); \
468 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l"); \
471 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q")); \
476 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
478 unsigned long _tmp; \
479 switch ((_dst).bytes) { \
481 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b"); \
484 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
485 _wx, _wy, _lx, _ly, _qx, _qy); \
490 /* Source operand is byte-sized and may be restricted to just %cl. */
491 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
492 __emulate_2op(_op, _src, _dst, _eflags, \
493 "b", "c", "b", "c", "b", "c", "b", "c")
495 /* Source operand is byte, word, long or quad sized. */
496 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
497 __emulate_2op(_op, _src, _dst, _eflags, \
498 "b", "q", "w", "r", _LO32, "r", "", "r")
500 /* Source operand is word, long or quad sized. */
501 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
502 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
503 "w", "r", _LO32, "r", "", "r")
505 /* Instruction has three operands and one operand is stored in ECX register */
506 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
508 unsigned long _tmp; \
509 _type _clv = (_cl).val; \
510 _type _srcv = (_src).val; \
511 _type _dstv = (_dst).val; \
513 __asm__ __volatile__ ( \
514 _PRE_EFLAGS("0", "5", "2") \
515 _op _suffix " %4,%1 \n" \
516 _POST_EFLAGS("0", "5", "2") \
517 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
518 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
521 (_cl).val = (unsigned long) _clv; \
522 (_src).val = (unsigned long) _srcv; \
523 (_dst).val = (unsigned long) _dstv; \
526 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
528 switch ((_dst).bytes) { \
530 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
531 "w", unsigned short); \
534 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
535 "l", unsigned int); \
538 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
539 "q", unsigned long)); \
544 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
546 unsigned long _tmp; \
548 __asm__ __volatile__ ( \
549 _PRE_EFLAGS("0", "3", "2") \
550 _op _suffix " %1; " \
551 _POST_EFLAGS("0", "3", "2") \
552 : "=m" (_eflags), "+m" ((_dst).val), \
554 : "i" (EFLAGS_MASK)); \
557 /* Instruction has only one explicit operand (no source operand). */
558 #define emulate_1op(_op, _dst, _eflags) \
560 switch ((_dst).bytes) { \
561 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
562 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
563 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
564 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
568 /* Fetch next part of the instruction being emulated. */
569 #define insn_fetch(_type, _size, _eip) \
570 ({ unsigned long _x; \
571 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
572 if (rc != X86EMUL_CONTINUE) \
578 #define insn_fetch_arr(_arr, _size, _eip) \
579 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
580 if (rc != X86EMUL_CONTINUE) \
585 static inline unsigned long ad_mask(struct decode_cache *c)
587 return (1UL << (c->ad_bytes << 3)) - 1;
590 /* Access/update address held in a register, based on addressing mode. */
591 static inline unsigned long
592 address_mask(struct decode_cache *c, unsigned long reg)
594 if (c->ad_bytes == sizeof(unsigned long))
597 return reg & ad_mask(c);
600 static inline unsigned long
601 register_address(struct decode_cache *c, unsigned long base, unsigned long reg)
603 return base + address_mask(c, reg);
607 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
609 if (c->ad_bytes == sizeof(unsigned long))
612 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
615 static inline void jmp_rel(struct decode_cache *c, int rel)
617 register_address_increment(c, &c->eip, rel);
620 static void set_seg_override(struct decode_cache *c, int seg)
622 c->has_seg_override = true;
623 c->seg_override = seg;
626 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
627 struct x86_emulate_ops *ops, int seg)
629 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
632 return ops->get_cached_segment_base(seg, ctxt->vcpu);
635 static unsigned long seg_override_base(struct x86_emulate_ctxt *ctxt,
636 struct x86_emulate_ops *ops,
637 struct decode_cache *c)
639 if (!c->has_seg_override)
642 return seg_base(ctxt, ops, c->seg_override);
645 static unsigned long es_base(struct x86_emulate_ctxt *ctxt,
646 struct x86_emulate_ops *ops)
648 return seg_base(ctxt, ops, VCPU_SREG_ES);
651 static unsigned long ss_base(struct x86_emulate_ctxt *ctxt,
652 struct x86_emulate_ops *ops)
654 return seg_base(ctxt, ops, VCPU_SREG_SS);
657 static void emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
658 u32 error, bool valid)
660 ctxt->exception = vec;
661 ctxt->error_code = error;
662 ctxt->error_code_valid = valid;
663 ctxt->restart = false;
666 static void emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
668 emulate_exception(ctxt, GP_VECTOR, err, true);
671 static void emulate_pf(struct x86_emulate_ctxt *ctxt, unsigned long addr,
675 emulate_exception(ctxt, PF_VECTOR, err, true);
678 static void emulate_ud(struct x86_emulate_ctxt *ctxt)
680 emulate_exception(ctxt, UD_VECTOR, 0, false);
683 static void emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
685 emulate_exception(ctxt, TS_VECTOR, err, true);
688 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
689 struct x86_emulate_ops *ops,
690 unsigned long eip, u8 *dest)
692 struct fetch_cache *fc = &ctxt->decode.fetch;
696 if (eip == fc->end) {
697 cur_size = fc->end - fc->start;
698 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
699 rc = ops->fetch(ctxt->cs_base + eip, fc->data + cur_size,
700 size, ctxt->vcpu, NULL);
701 if (rc != X86EMUL_CONTINUE)
705 *dest = fc->data[eip - fc->start];
706 return X86EMUL_CONTINUE;
709 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
710 struct x86_emulate_ops *ops,
711 unsigned long eip, void *dest, unsigned size)
715 /* x86 instructions are limited to 15 bytes. */
716 if (eip + size - ctxt->eip > 15)
717 return X86EMUL_UNHANDLEABLE;
719 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
720 if (rc != X86EMUL_CONTINUE)
723 return X86EMUL_CONTINUE;
727 * Given the 'reg' portion of a ModRM byte, and a register block, return a
728 * pointer into the block that addresses the relevant register.
729 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
731 static void *decode_register(u8 modrm_reg, unsigned long *regs,
736 p = ®s[modrm_reg];
737 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
738 p = (unsigned char *)®s[modrm_reg & 3] + 1;
742 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
743 struct x86_emulate_ops *ops,
745 u16 *size, unsigned long *address, int op_bytes)
752 rc = ops->read_std((unsigned long)ptr, (unsigned long *)size, 2,
754 if (rc != X86EMUL_CONTINUE)
756 rc = ops->read_std((unsigned long)ptr + 2, address, op_bytes,
761 static int test_cc(unsigned int condition, unsigned int flags)
765 switch ((condition & 15) >> 1) {
767 rc |= (flags & EFLG_OF);
769 case 1: /* b/c/nae */
770 rc |= (flags & EFLG_CF);
773 rc |= (flags & EFLG_ZF);
776 rc |= (flags & (EFLG_CF|EFLG_ZF));
779 rc |= (flags & EFLG_SF);
782 rc |= (flags & EFLG_PF);
785 rc |= (flags & EFLG_ZF);
788 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
792 /* Odd condition identifiers (lsb == 1) have inverted sense. */
793 return (!!rc ^ (condition & 1));
796 static void decode_register_operand(struct operand *op,
797 struct decode_cache *c,
800 unsigned reg = c->modrm_reg;
801 int highbyte_regs = c->rex_prefix == 0;
804 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
806 if ((c->d & ByteOp) && !inhibit_bytereg) {
807 op->ptr = decode_register(reg, c->regs, highbyte_regs);
808 op->val = *(u8 *)op->ptr;
811 op->ptr = decode_register(reg, c->regs, 0);
812 op->bytes = c->op_bytes;
815 op->val = *(u16 *)op->ptr;
818 op->val = *(u32 *)op->ptr;
821 op->val = *(u64 *) op->ptr;
825 op->orig_val = op->val;
828 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
829 struct x86_emulate_ops *ops)
831 struct decode_cache *c = &ctxt->decode;
833 int index_reg = 0, base_reg = 0, scale;
834 int rc = X86EMUL_CONTINUE;
837 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
838 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
839 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
842 c->modrm = insn_fetch(u8, 1, c->eip);
843 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
844 c->modrm_reg |= (c->modrm & 0x38) >> 3;
845 c->modrm_rm |= (c->modrm & 0x07);
849 if (c->modrm_mod == 3) {
850 c->modrm_ptr = decode_register(c->modrm_rm,
851 c->regs, c->d & ByteOp);
852 c->modrm_val = *(unsigned long *)c->modrm_ptr;
856 if (c->ad_bytes == 2) {
857 unsigned bx = c->regs[VCPU_REGS_RBX];
858 unsigned bp = c->regs[VCPU_REGS_RBP];
859 unsigned si = c->regs[VCPU_REGS_RSI];
860 unsigned di = c->regs[VCPU_REGS_RDI];
862 /* 16-bit ModR/M decode. */
863 switch (c->modrm_mod) {
865 if (c->modrm_rm == 6)
866 c->modrm_ea += insn_fetch(u16, 2, c->eip);
869 c->modrm_ea += insn_fetch(s8, 1, c->eip);
872 c->modrm_ea += insn_fetch(u16, 2, c->eip);
875 switch (c->modrm_rm) {
877 c->modrm_ea += bx + si;
880 c->modrm_ea += bx + di;
883 c->modrm_ea += bp + si;
886 c->modrm_ea += bp + di;
895 if (c->modrm_mod != 0)
902 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
903 (c->modrm_rm == 6 && c->modrm_mod != 0))
904 if (!c->has_seg_override)
905 set_seg_override(c, VCPU_SREG_SS);
906 c->modrm_ea = (u16)c->modrm_ea;
908 /* 32/64-bit ModR/M decode. */
909 if ((c->modrm_rm & 7) == 4) {
910 sib = insn_fetch(u8, 1, c->eip);
911 index_reg |= (sib >> 3) & 7;
915 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
916 c->modrm_ea += insn_fetch(s32, 4, c->eip);
918 c->modrm_ea += c->regs[base_reg];
920 c->modrm_ea += c->regs[index_reg] << scale;
921 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
922 if (ctxt->mode == X86EMUL_MODE_PROT64)
925 c->modrm_ea += c->regs[c->modrm_rm];
926 switch (c->modrm_mod) {
928 if (c->modrm_rm == 5)
929 c->modrm_ea += insn_fetch(s32, 4, c->eip);
932 c->modrm_ea += insn_fetch(s8, 1, c->eip);
935 c->modrm_ea += insn_fetch(s32, 4, c->eip);
943 static int decode_abs(struct x86_emulate_ctxt *ctxt,
944 struct x86_emulate_ops *ops)
946 struct decode_cache *c = &ctxt->decode;
947 int rc = X86EMUL_CONTINUE;
949 switch (c->ad_bytes) {
951 c->modrm_ea = insn_fetch(u16, 2, c->eip);
954 c->modrm_ea = insn_fetch(u32, 4, c->eip);
957 c->modrm_ea = insn_fetch(u64, 8, c->eip);
965 x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
967 struct decode_cache *c = &ctxt->decode;
968 int rc = X86EMUL_CONTINUE;
969 int mode = ctxt->mode;
970 int def_op_bytes, def_ad_bytes, group;
973 /* we cannot decode insn before we complete previous rep insn */
974 WARN_ON(ctxt->restart);
977 c->fetch.start = c->fetch.end = c->eip;
978 ctxt->cs_base = seg_base(ctxt, ops, VCPU_SREG_CS);
981 case X86EMUL_MODE_REAL:
982 case X86EMUL_MODE_VM86:
983 case X86EMUL_MODE_PROT16:
984 def_op_bytes = def_ad_bytes = 2;
986 case X86EMUL_MODE_PROT32:
987 def_op_bytes = def_ad_bytes = 4;
990 case X86EMUL_MODE_PROT64:
999 c->op_bytes = def_op_bytes;
1000 c->ad_bytes = def_ad_bytes;
1002 /* Legacy prefixes. */
1004 switch (c->b = insn_fetch(u8, 1, c->eip)) {
1005 case 0x66: /* operand-size override */
1006 /* switch between 2/4 bytes */
1007 c->op_bytes = def_op_bytes ^ 6;
1009 case 0x67: /* address-size override */
1010 if (mode == X86EMUL_MODE_PROT64)
1011 /* switch between 4/8 bytes */
1012 c->ad_bytes = def_ad_bytes ^ 12;
1014 /* switch between 2/4 bytes */
1015 c->ad_bytes = def_ad_bytes ^ 6;
1017 case 0x26: /* ES override */
1018 case 0x2e: /* CS override */
1019 case 0x36: /* SS override */
1020 case 0x3e: /* DS override */
1021 set_seg_override(c, (c->b >> 3) & 3);
1023 case 0x64: /* FS override */
1024 case 0x65: /* GS override */
1025 set_seg_override(c, c->b & 7);
1027 case 0x40 ... 0x4f: /* REX */
1028 if (mode != X86EMUL_MODE_PROT64)
1030 c->rex_prefix = c->b;
1032 case 0xf0: /* LOCK */
1035 case 0xf2: /* REPNE/REPNZ */
1036 c->rep_prefix = REPNE_PREFIX;
1038 case 0xf3: /* REP/REPE/REPZ */
1039 c->rep_prefix = REPE_PREFIX;
1045 /* Any legacy prefix after a REX prefix nullifies its effect. */
1054 if (c->rex_prefix & 8)
1055 c->op_bytes = 8; /* REX.W */
1057 /* Opcode byte(s). */
1058 c->d = opcode_table[c->b];
1060 /* Two-byte opcode? */
1063 c->b = insn_fetch(u8, 1, c->eip);
1064 c->d = twobyte_table[c->b];
1069 group = c->d & GroupMask;
1070 c->modrm = insn_fetch(u8, 1, c->eip);
1073 group = (group << 3) + ((c->modrm >> 3) & 7);
1074 if ((c->d & GroupDual) && (c->modrm >> 6) == 3)
1075 c->d = group2_table[group];
1077 c->d = group_table[group];
1082 DPRINTF("Cannot emulate %02x\n", c->b);
1086 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
1089 /* ModRM and SIB bytes. */
1091 rc = decode_modrm(ctxt, ops);
1092 else if (c->d & MemAbs)
1093 rc = decode_abs(ctxt, ops);
1094 if (rc != X86EMUL_CONTINUE)
1097 if (!c->has_seg_override)
1098 set_seg_override(c, VCPU_SREG_DS);
1100 if (!(!c->twobyte && c->b == 0x8d))
1101 c->modrm_ea += seg_override_base(ctxt, ops, c);
1103 if (c->ad_bytes != 8)
1104 c->modrm_ea = (u32)c->modrm_ea;
1106 if (c->rip_relative)
1107 c->modrm_ea += c->eip;
1110 * Decode and fetch the source operand: register, memory
1113 switch (c->d & SrcMask) {
1117 decode_register_operand(&c->src, c, 0);
1126 c->src.bytes = (c->d & ByteOp) ? 1 :
1128 /* Don't fetch the address for invlpg: it could be unmapped. */
1129 if (c->twobyte && c->b == 0x01 && c->modrm_reg == 7)
1133 * For instructions with a ModR/M byte, switch to register
1134 * access if Mod = 3.
1136 if ((c->d & ModRM) && c->modrm_mod == 3) {
1137 c->src.type = OP_REG;
1138 c->src.val = c->modrm_val;
1139 c->src.ptr = c->modrm_ptr;
1142 c->src.type = OP_MEM;
1143 c->src.ptr = (unsigned long *)c->modrm_ea;
1148 c->src.type = OP_IMM;
1149 c->src.ptr = (unsigned long *)c->eip;
1150 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1151 if (c->src.bytes == 8)
1153 /* NB. Immediates are sign-extended as necessary. */
1154 switch (c->src.bytes) {
1156 c->src.val = insn_fetch(s8, 1, c->eip);
1159 c->src.val = insn_fetch(s16, 2, c->eip);
1162 c->src.val = insn_fetch(s32, 4, c->eip);
1165 if ((c->d & SrcMask) == SrcImmU) {
1166 switch (c->src.bytes) {
1171 c->src.val &= 0xffff;
1174 c->src.val &= 0xffffffff;
1181 c->src.type = OP_IMM;
1182 c->src.ptr = (unsigned long *)c->eip;
1184 if ((c->d & SrcMask) == SrcImmByte)
1185 c->src.val = insn_fetch(s8, 1, c->eip);
1187 c->src.val = insn_fetch(u8, 1, c->eip);
1194 c->src.type = OP_MEM;
1195 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1196 c->src.ptr = (unsigned long *)
1197 register_address(c, seg_override_base(ctxt, ops, c),
1198 c->regs[VCPU_REGS_RSI]);
1202 c->src.type = OP_IMM;
1203 c->src.ptr = (unsigned long *)c->eip;
1204 c->src.bytes = c->op_bytes + 2;
1205 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
1208 c->src.type = OP_MEM;
1209 c->src.ptr = (unsigned long *)c->modrm_ea;
1210 c->src.bytes = c->op_bytes + 2;
1215 * Decode and fetch the second source operand: register, memory
1218 switch (c->d & Src2Mask) {
1223 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
1226 c->src2.type = OP_IMM;
1227 c->src2.ptr = (unsigned long *)c->eip;
1229 c->src2.val = insn_fetch(u8, 1, c->eip);
1237 /* Decode and fetch the destination operand: register or memory. */
1238 switch (c->d & DstMask) {
1240 /* Special instructions do their own operand decoding. */
1243 decode_register_operand(&c->dst, c,
1244 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
1248 if ((c->d & ModRM) && c->modrm_mod == 3) {
1249 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1250 c->dst.type = OP_REG;
1251 c->dst.val = c->dst.orig_val = c->modrm_val;
1252 c->dst.ptr = c->modrm_ptr;
1255 c->dst.type = OP_MEM;
1256 c->dst.ptr = (unsigned long *)c->modrm_ea;
1257 if ((c->d & DstMask) == DstMem64)
1260 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1263 unsigned long mask = ~(c->dst.bytes * 8 - 1);
1265 c->dst.ptr = (void *)c->dst.ptr +
1266 (c->src.val & mask) / 8;
1270 c->dst.type = OP_REG;
1271 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1272 c->dst.ptr = &c->regs[VCPU_REGS_RAX];
1273 switch (c->dst.bytes) {
1275 c->dst.val = *(u8 *)c->dst.ptr;
1278 c->dst.val = *(u16 *)c->dst.ptr;
1281 c->dst.val = *(u32 *)c->dst.ptr;
1284 c->dst.val = *(u64 *)c->dst.ptr;
1287 c->dst.orig_val = c->dst.val;
1290 c->dst.type = OP_MEM;
1291 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1292 c->dst.ptr = (unsigned long *)
1293 register_address(c, es_base(ctxt, ops),
1294 c->regs[VCPU_REGS_RDI]);
1300 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
1303 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1304 struct x86_emulate_ops *ops,
1305 unsigned long addr, void *dest, unsigned size)
1308 struct read_cache *mc = &ctxt->decode.mem_read;
1312 int n = min(size, 8u);
1314 if (mc->pos < mc->end)
1317 rc = ops->read_emulated(addr, mc->data + mc->end, n, &err,
1319 if (rc == X86EMUL_PROPAGATE_FAULT)
1320 emulate_pf(ctxt, addr, err);
1321 if (rc != X86EMUL_CONTINUE)
1326 memcpy(dest, mc->data + mc->pos, n);
1331 return X86EMUL_CONTINUE;
1334 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1335 struct x86_emulate_ops *ops,
1336 unsigned int size, unsigned short port,
1339 struct read_cache *rc = &ctxt->decode.io_read;
1341 if (rc->pos == rc->end) { /* refill pio read ahead */
1342 struct decode_cache *c = &ctxt->decode;
1343 unsigned int in_page, n;
1344 unsigned int count = c->rep_prefix ?
1345 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
1346 in_page = (ctxt->eflags & EFLG_DF) ?
1347 offset_in_page(c->regs[VCPU_REGS_RDI]) :
1348 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
1349 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1353 rc->pos = rc->end = 0;
1354 if (!ops->pio_in_emulated(size, port, rc->data, n, ctxt->vcpu))
1359 memcpy(dest, rc->data + rc->pos, size);
1364 static u32 desc_limit_scaled(struct desc_struct *desc)
1366 u32 limit = get_desc_limit(desc);
1368 return desc->g ? (limit << 12) | 0xfff : limit;
1371 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1372 struct x86_emulate_ops *ops,
1373 u16 selector, struct desc_ptr *dt)
1375 if (selector & 1 << 2) {
1376 struct desc_struct desc;
1377 memset (dt, 0, sizeof *dt);
1378 if (!ops->get_cached_descriptor(&desc, VCPU_SREG_LDTR, ctxt->vcpu))
1381 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1382 dt->address = get_desc_base(&desc);
1384 ops->get_gdt(dt, ctxt->vcpu);
1387 /* allowed just for 8 bytes segments */
1388 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1389 struct x86_emulate_ops *ops,
1390 u16 selector, struct desc_struct *desc)
1393 u16 index = selector >> 3;
1398 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1400 if (dt.size < index * 8 + 7) {
1401 emulate_gp(ctxt, selector & 0xfffc);
1402 return X86EMUL_PROPAGATE_FAULT;
1404 addr = dt.address + index * 8;
1405 ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu, &err);
1406 if (ret == X86EMUL_PROPAGATE_FAULT)
1407 emulate_pf(ctxt, addr, err);
1412 /* allowed just for 8 bytes segments */
1413 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1414 struct x86_emulate_ops *ops,
1415 u16 selector, struct desc_struct *desc)
1418 u16 index = selector >> 3;
1423 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1425 if (dt.size < index * 8 + 7) {
1426 emulate_gp(ctxt, selector & 0xfffc);
1427 return X86EMUL_PROPAGATE_FAULT;
1430 addr = dt.address + index * 8;
1431 ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu, &err);
1432 if (ret == X86EMUL_PROPAGATE_FAULT)
1433 emulate_pf(ctxt, addr, err);
1438 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1439 struct x86_emulate_ops *ops,
1440 u16 selector, int seg)
1442 struct desc_struct seg_desc;
1444 unsigned err_vec = GP_VECTOR;
1446 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1449 memset(&seg_desc, 0, sizeof seg_desc);
1451 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1452 || ctxt->mode == X86EMUL_MODE_REAL) {
1453 /* set real mode segment descriptor */
1454 set_desc_base(&seg_desc, selector << 4);
1455 set_desc_limit(&seg_desc, 0xffff);
1462 /* NULL selector is not valid for TR, CS and SS */
1463 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1467 /* TR should be in GDT only */
1468 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1471 if (null_selector) /* for NULL selector skip all following checks */
1474 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1475 if (ret != X86EMUL_CONTINUE)
1478 err_code = selector & 0xfffc;
1479 err_vec = GP_VECTOR;
1481 /* can't load system descriptor into segment selecor */
1482 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1486 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1492 cpl = ops->cpl(ctxt->vcpu);
1497 * segment is not a writable data segment or segment
1498 * selector's RPL != CPL or segment selector's RPL != CPL
1500 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1504 if (!(seg_desc.type & 8))
1507 if (seg_desc.type & 4) {
1513 if (rpl > cpl || dpl != cpl)
1516 /* CS(RPL) <- CPL */
1517 selector = (selector & 0xfffc) | cpl;
1520 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1523 case VCPU_SREG_LDTR:
1524 if (seg_desc.s || seg_desc.type != 2)
1527 default: /* DS, ES, FS, or GS */
1529 * segment is not a data or readable code segment or
1530 * ((segment is a data or nonconforming code segment)
1531 * and (both RPL and CPL > DPL))
1533 if ((seg_desc.type & 0xa) == 0x8 ||
1534 (((seg_desc.type & 0xc) != 0xc) &&
1535 (rpl > dpl && cpl > dpl)))
1541 /* mark segment as accessed */
1543 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1544 if (ret != X86EMUL_CONTINUE)
1548 ops->set_segment_selector(selector, seg, ctxt->vcpu);
1549 ops->set_cached_descriptor(&seg_desc, seg, ctxt->vcpu);
1550 return X86EMUL_CONTINUE;
1552 emulate_exception(ctxt, err_vec, err_code, true);
1553 return X86EMUL_PROPAGATE_FAULT;
1556 static inline void emulate_push(struct x86_emulate_ctxt *ctxt,
1557 struct x86_emulate_ops *ops)
1559 struct decode_cache *c = &ctxt->decode;
1561 c->dst.type = OP_MEM;
1562 c->dst.bytes = c->op_bytes;
1563 c->dst.val = c->src.val;
1564 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1565 c->dst.ptr = (void *) register_address(c, ss_base(ctxt, ops),
1566 c->regs[VCPU_REGS_RSP]);
1569 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1570 struct x86_emulate_ops *ops,
1571 void *dest, int len)
1573 struct decode_cache *c = &ctxt->decode;
1576 rc = read_emulated(ctxt, ops, register_address(c, ss_base(ctxt, ops),
1577 c->regs[VCPU_REGS_RSP]),
1579 if (rc != X86EMUL_CONTINUE)
1582 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1586 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1587 struct x86_emulate_ops *ops,
1588 void *dest, int len)
1591 unsigned long val, change_mask;
1592 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1593 int cpl = ops->cpl(ctxt->vcpu);
1595 rc = emulate_pop(ctxt, ops, &val, len);
1596 if (rc != X86EMUL_CONTINUE)
1599 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1600 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1602 switch(ctxt->mode) {
1603 case X86EMUL_MODE_PROT64:
1604 case X86EMUL_MODE_PROT32:
1605 case X86EMUL_MODE_PROT16:
1607 change_mask |= EFLG_IOPL;
1609 change_mask |= EFLG_IF;
1611 case X86EMUL_MODE_VM86:
1613 emulate_gp(ctxt, 0);
1614 return X86EMUL_PROPAGATE_FAULT;
1616 change_mask |= EFLG_IF;
1618 default: /* real mode */
1619 change_mask |= (EFLG_IOPL | EFLG_IF);
1623 *(unsigned long *)dest =
1624 (ctxt->eflags & ~change_mask) | (val & change_mask);
1629 static void emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1630 struct x86_emulate_ops *ops, int seg)
1632 struct decode_cache *c = &ctxt->decode;
1634 c->src.val = ops->get_segment_selector(seg, ctxt->vcpu);
1636 emulate_push(ctxt, ops);
1639 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1640 struct x86_emulate_ops *ops, int seg)
1642 struct decode_cache *c = &ctxt->decode;
1643 unsigned long selector;
1646 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1647 if (rc != X86EMUL_CONTINUE)
1650 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1654 static void emulate_pusha(struct x86_emulate_ctxt *ctxt,
1655 struct x86_emulate_ops *ops)
1657 struct decode_cache *c = &ctxt->decode;
1658 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1659 int reg = VCPU_REGS_RAX;
1661 while (reg <= VCPU_REGS_RDI) {
1662 (reg == VCPU_REGS_RSP) ?
1663 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1665 emulate_push(ctxt, ops);
1670 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1671 struct x86_emulate_ops *ops)
1673 struct decode_cache *c = &ctxt->decode;
1674 int rc = X86EMUL_CONTINUE;
1675 int reg = VCPU_REGS_RDI;
1677 while (reg >= VCPU_REGS_RAX) {
1678 if (reg == VCPU_REGS_RSP) {
1679 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1684 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1685 if (rc != X86EMUL_CONTINUE)
1692 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1693 struct x86_emulate_ops *ops)
1695 struct decode_cache *c = &ctxt->decode;
1697 return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1700 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1702 struct decode_cache *c = &ctxt->decode;
1703 switch (c->modrm_reg) {
1705 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1708 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1711 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1714 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1716 case 4: /* sal/shl */
1717 case 6: /* sal/shl */
1718 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1721 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1724 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1729 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1730 struct x86_emulate_ops *ops)
1732 struct decode_cache *c = &ctxt->decode;
1734 switch (c->modrm_reg) {
1735 case 0 ... 1: /* test */
1736 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1739 c->dst.val = ~c->dst.val;
1742 emulate_1op("neg", c->dst, ctxt->eflags);
1750 static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
1751 struct x86_emulate_ops *ops)
1753 struct decode_cache *c = &ctxt->decode;
1755 switch (c->modrm_reg) {
1757 emulate_1op("inc", c->dst, ctxt->eflags);
1760 emulate_1op("dec", c->dst, ctxt->eflags);
1762 case 2: /* call near abs */ {
1765 c->eip = c->src.val;
1766 c->src.val = old_eip;
1767 emulate_push(ctxt, ops);
1770 case 4: /* jmp abs */
1771 c->eip = c->src.val;
1774 emulate_push(ctxt, ops);
1777 return X86EMUL_CONTINUE;
1780 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1781 struct x86_emulate_ops *ops)
1783 struct decode_cache *c = &ctxt->decode;
1784 u64 old = c->dst.orig_val;
1786 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1787 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1789 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1790 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1791 ctxt->eflags &= ~EFLG_ZF;
1793 c->dst.val = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1794 (u32) c->regs[VCPU_REGS_RBX];
1796 ctxt->eflags |= EFLG_ZF;
1798 return X86EMUL_CONTINUE;
1801 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1802 struct x86_emulate_ops *ops)
1804 struct decode_cache *c = &ctxt->decode;
1808 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1809 if (rc != X86EMUL_CONTINUE)
1811 if (c->op_bytes == 4)
1812 c->eip = (u32)c->eip;
1813 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1814 if (rc != X86EMUL_CONTINUE)
1816 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1820 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1821 struct x86_emulate_ops *ops)
1824 struct decode_cache *c = &ctxt->decode;
1827 switch (c->dst.type) {
1829 /* The 4-byte case *is* correct:
1830 * in 64-bit mode we zero-extend.
1832 switch (c->dst.bytes) {
1834 *(u8 *)c->dst.ptr = (u8)c->dst.val;
1837 *(u16 *)c->dst.ptr = (u16)c->dst.val;
1840 *c->dst.ptr = (u32)c->dst.val;
1841 break; /* 64b: zero-ext */
1843 *c->dst.ptr = c->dst.val;
1849 rc = ops->cmpxchg_emulated(
1850 (unsigned long)c->dst.ptr,
1857 rc = ops->write_emulated(
1858 (unsigned long)c->dst.ptr,
1863 if (rc == X86EMUL_PROPAGATE_FAULT)
1865 (unsigned long)c->dst.ptr, err);
1866 if (rc != X86EMUL_CONTINUE)
1875 return X86EMUL_CONTINUE;
1879 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1880 struct x86_emulate_ops *ops, struct desc_struct *cs,
1881 struct desc_struct *ss)
1883 memset(cs, 0, sizeof(struct desc_struct));
1884 ops->get_cached_descriptor(cs, VCPU_SREG_CS, ctxt->vcpu);
1885 memset(ss, 0, sizeof(struct desc_struct));
1887 cs->l = 0; /* will be adjusted later */
1888 set_desc_base(cs, 0); /* flat segment */
1889 cs->g = 1; /* 4kb granularity */
1890 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1891 cs->type = 0x0b; /* Read, Execute, Accessed */
1893 cs->dpl = 0; /* will be adjusted later */
1897 set_desc_base(ss, 0); /* flat segment */
1898 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1899 ss->g = 1; /* 4kb granularity */
1901 ss->type = 0x03; /* Read/Write, Accessed */
1902 ss->d = 1; /* 32bit stack segment */
1908 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1910 struct decode_cache *c = &ctxt->decode;
1911 struct desc_struct cs, ss;
1915 /* syscall is not available in real mode */
1916 if (ctxt->mode == X86EMUL_MODE_REAL ||
1917 ctxt->mode == X86EMUL_MODE_VM86) {
1919 return X86EMUL_PROPAGATE_FAULT;
1922 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1924 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1926 cs_sel = (u16)(msr_data & 0xfffc);
1927 ss_sel = (u16)(msr_data + 8);
1929 if (is_long_mode(ctxt->vcpu)) {
1933 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
1934 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1935 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
1936 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1938 c->regs[VCPU_REGS_RCX] = c->eip;
1939 if (is_long_mode(ctxt->vcpu)) {
1940 #ifdef CONFIG_X86_64
1941 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1943 ops->get_msr(ctxt->vcpu,
1944 ctxt->mode == X86EMUL_MODE_PROT64 ?
1945 MSR_LSTAR : MSR_CSTAR, &msr_data);
1948 ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
1949 ctxt->eflags &= ~(msr_data | EFLG_RF);
1953 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1954 c->eip = (u32)msr_data;
1956 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1959 return X86EMUL_CONTINUE;
1963 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1965 struct decode_cache *c = &ctxt->decode;
1966 struct desc_struct cs, ss;
1970 /* inject #GP if in real mode */
1971 if (ctxt->mode == X86EMUL_MODE_REAL) {
1972 emulate_gp(ctxt, 0);
1973 return X86EMUL_PROPAGATE_FAULT;
1976 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1977 * Therefore, we inject an #UD.
1979 if (ctxt->mode == X86EMUL_MODE_PROT64) {
1981 return X86EMUL_PROPAGATE_FAULT;
1984 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1986 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1987 switch (ctxt->mode) {
1988 case X86EMUL_MODE_PROT32:
1989 if ((msr_data & 0xfffc) == 0x0) {
1990 emulate_gp(ctxt, 0);
1991 return X86EMUL_PROPAGATE_FAULT;
1994 case X86EMUL_MODE_PROT64:
1995 if (msr_data == 0x0) {
1996 emulate_gp(ctxt, 0);
1997 return X86EMUL_PROPAGATE_FAULT;
2002 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2003 cs_sel = (u16)msr_data;
2004 cs_sel &= ~SELECTOR_RPL_MASK;
2005 ss_sel = cs_sel + 8;
2006 ss_sel &= ~SELECTOR_RPL_MASK;
2007 if (ctxt->mode == X86EMUL_MODE_PROT64
2008 || is_long_mode(ctxt->vcpu)) {
2013 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
2014 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
2015 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
2016 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
2018 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
2021 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
2022 c->regs[VCPU_REGS_RSP] = msr_data;
2024 return X86EMUL_CONTINUE;
2028 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2030 struct decode_cache *c = &ctxt->decode;
2031 struct desc_struct cs, ss;
2036 /* inject #GP if in real mode or Virtual 8086 mode */
2037 if (ctxt->mode == X86EMUL_MODE_REAL ||
2038 ctxt->mode == X86EMUL_MODE_VM86) {
2039 emulate_gp(ctxt, 0);
2040 return X86EMUL_PROPAGATE_FAULT;
2043 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2045 if ((c->rex_prefix & 0x8) != 0x0)
2046 usermode = X86EMUL_MODE_PROT64;
2048 usermode = X86EMUL_MODE_PROT32;
2052 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
2054 case X86EMUL_MODE_PROT32:
2055 cs_sel = (u16)(msr_data + 16);
2056 if ((msr_data & 0xfffc) == 0x0) {
2057 emulate_gp(ctxt, 0);
2058 return X86EMUL_PROPAGATE_FAULT;
2060 ss_sel = (u16)(msr_data + 24);
2062 case X86EMUL_MODE_PROT64:
2063 cs_sel = (u16)(msr_data + 32);
2064 if (msr_data == 0x0) {
2065 emulate_gp(ctxt, 0);
2066 return X86EMUL_PROPAGATE_FAULT;
2068 ss_sel = cs_sel + 8;
2073 cs_sel |= SELECTOR_RPL_MASK;
2074 ss_sel |= SELECTOR_RPL_MASK;
2076 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
2077 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
2078 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
2079 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
2081 c->eip = c->regs[VCPU_REGS_RDX];
2082 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
2084 return X86EMUL_CONTINUE;
2087 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
2088 struct x86_emulate_ops *ops)
2091 if (ctxt->mode == X86EMUL_MODE_REAL)
2093 if (ctxt->mode == X86EMUL_MODE_VM86)
2095 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2096 return ops->cpl(ctxt->vcpu) > iopl;
2099 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2100 struct x86_emulate_ops *ops,
2103 struct desc_struct tr_seg;
2106 u8 perm, bit_idx = port & 0x7;
2107 unsigned mask = (1 << len) - 1;
2109 ops->get_cached_descriptor(&tr_seg, VCPU_SREG_TR, ctxt->vcpu);
2112 if (desc_limit_scaled(&tr_seg) < 103)
2114 r = ops->read_std(get_desc_base(&tr_seg) + 102, &io_bitmap_ptr, 2,
2116 if (r != X86EMUL_CONTINUE)
2118 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2120 r = ops->read_std(get_desc_base(&tr_seg) + io_bitmap_ptr + port/8,
2121 &perm, 1, ctxt->vcpu, NULL);
2122 if (r != X86EMUL_CONTINUE)
2124 if ((perm >> bit_idx) & mask)
2129 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2130 struct x86_emulate_ops *ops,
2133 if (emulator_bad_iopl(ctxt, ops))
2134 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
2139 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2140 struct x86_emulate_ops *ops,
2141 struct tss_segment_16 *tss)
2143 struct decode_cache *c = &ctxt->decode;
2146 tss->flag = ctxt->eflags;
2147 tss->ax = c->regs[VCPU_REGS_RAX];
2148 tss->cx = c->regs[VCPU_REGS_RCX];
2149 tss->dx = c->regs[VCPU_REGS_RDX];
2150 tss->bx = c->regs[VCPU_REGS_RBX];
2151 tss->sp = c->regs[VCPU_REGS_RSP];
2152 tss->bp = c->regs[VCPU_REGS_RBP];
2153 tss->si = c->regs[VCPU_REGS_RSI];
2154 tss->di = c->regs[VCPU_REGS_RDI];
2156 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2157 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2158 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2159 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2160 tss->ldt = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2163 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2164 struct x86_emulate_ops *ops,
2165 struct tss_segment_16 *tss)
2167 struct decode_cache *c = &ctxt->decode;
2171 ctxt->eflags = tss->flag | 2;
2172 c->regs[VCPU_REGS_RAX] = tss->ax;
2173 c->regs[VCPU_REGS_RCX] = tss->cx;
2174 c->regs[VCPU_REGS_RDX] = tss->dx;
2175 c->regs[VCPU_REGS_RBX] = tss->bx;
2176 c->regs[VCPU_REGS_RSP] = tss->sp;
2177 c->regs[VCPU_REGS_RBP] = tss->bp;
2178 c->regs[VCPU_REGS_RSI] = tss->si;
2179 c->regs[VCPU_REGS_RDI] = tss->di;
2182 * SDM says that segment selectors are loaded before segment
2185 ops->set_segment_selector(tss->ldt, VCPU_SREG_LDTR, ctxt->vcpu);
2186 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2187 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2188 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2189 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2192 * Now load segment descriptors. If fault happenes at this stage
2193 * it is handled in a context of new task
2195 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
2196 if (ret != X86EMUL_CONTINUE)
2198 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2199 if (ret != X86EMUL_CONTINUE)
2201 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2202 if (ret != X86EMUL_CONTINUE)
2204 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2205 if (ret != X86EMUL_CONTINUE)
2207 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2208 if (ret != X86EMUL_CONTINUE)
2211 return X86EMUL_CONTINUE;
2214 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2215 struct x86_emulate_ops *ops,
2216 u16 tss_selector, u16 old_tss_sel,
2217 ulong old_tss_base, struct desc_struct *new_desc)
2219 struct tss_segment_16 tss_seg;
2221 u32 err, new_tss_base = get_desc_base(new_desc);
2223 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2225 if (ret == X86EMUL_PROPAGATE_FAULT) {
2226 /* FIXME: need to provide precise fault address */
2227 emulate_pf(ctxt, old_tss_base, err);
2231 save_state_to_tss16(ctxt, ops, &tss_seg);
2233 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2235 if (ret == X86EMUL_PROPAGATE_FAULT) {
2236 /* FIXME: need to provide precise fault address */
2237 emulate_pf(ctxt, old_tss_base, err);
2241 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2243 if (ret == X86EMUL_PROPAGATE_FAULT) {
2244 /* FIXME: need to provide precise fault address */
2245 emulate_pf(ctxt, new_tss_base, err);
2249 if (old_tss_sel != 0xffff) {
2250 tss_seg.prev_task_link = old_tss_sel;
2252 ret = ops->write_std(new_tss_base,
2253 &tss_seg.prev_task_link,
2254 sizeof tss_seg.prev_task_link,
2256 if (ret == X86EMUL_PROPAGATE_FAULT) {
2257 /* FIXME: need to provide precise fault address */
2258 emulate_pf(ctxt, new_tss_base, err);
2263 return load_state_from_tss16(ctxt, ops, &tss_seg);
2266 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2267 struct x86_emulate_ops *ops,
2268 struct tss_segment_32 *tss)
2270 struct decode_cache *c = &ctxt->decode;
2272 tss->cr3 = ops->get_cr(3, ctxt->vcpu);
2274 tss->eflags = ctxt->eflags;
2275 tss->eax = c->regs[VCPU_REGS_RAX];
2276 tss->ecx = c->regs[VCPU_REGS_RCX];
2277 tss->edx = c->regs[VCPU_REGS_RDX];
2278 tss->ebx = c->regs[VCPU_REGS_RBX];
2279 tss->esp = c->regs[VCPU_REGS_RSP];
2280 tss->ebp = c->regs[VCPU_REGS_RBP];
2281 tss->esi = c->regs[VCPU_REGS_RSI];
2282 tss->edi = c->regs[VCPU_REGS_RDI];
2284 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2285 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2286 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2287 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2288 tss->fs = ops->get_segment_selector(VCPU_SREG_FS, ctxt->vcpu);
2289 tss->gs = ops->get_segment_selector(VCPU_SREG_GS, ctxt->vcpu);
2290 tss->ldt_selector = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2293 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2294 struct x86_emulate_ops *ops,
2295 struct tss_segment_32 *tss)
2297 struct decode_cache *c = &ctxt->decode;
2300 if (ops->set_cr(3, tss->cr3, ctxt->vcpu)) {
2301 emulate_gp(ctxt, 0);
2302 return X86EMUL_PROPAGATE_FAULT;
2305 ctxt->eflags = tss->eflags | 2;
2306 c->regs[VCPU_REGS_RAX] = tss->eax;
2307 c->regs[VCPU_REGS_RCX] = tss->ecx;
2308 c->regs[VCPU_REGS_RDX] = tss->edx;
2309 c->regs[VCPU_REGS_RBX] = tss->ebx;
2310 c->regs[VCPU_REGS_RSP] = tss->esp;
2311 c->regs[VCPU_REGS_RBP] = tss->ebp;
2312 c->regs[VCPU_REGS_RSI] = tss->esi;
2313 c->regs[VCPU_REGS_RDI] = tss->edi;
2316 * SDM says that segment selectors are loaded before segment
2319 ops->set_segment_selector(tss->ldt_selector, VCPU_SREG_LDTR, ctxt->vcpu);
2320 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2321 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2322 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2323 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2324 ops->set_segment_selector(tss->fs, VCPU_SREG_FS, ctxt->vcpu);
2325 ops->set_segment_selector(tss->gs, VCPU_SREG_GS, ctxt->vcpu);
2328 * Now load segment descriptors. If fault happenes at this stage
2329 * it is handled in a context of new task
2331 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2332 if (ret != X86EMUL_CONTINUE)
2334 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2335 if (ret != X86EMUL_CONTINUE)
2337 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2338 if (ret != X86EMUL_CONTINUE)
2340 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2341 if (ret != X86EMUL_CONTINUE)
2343 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2344 if (ret != X86EMUL_CONTINUE)
2346 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2347 if (ret != X86EMUL_CONTINUE)
2349 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2350 if (ret != X86EMUL_CONTINUE)
2353 return X86EMUL_CONTINUE;
2356 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2357 struct x86_emulate_ops *ops,
2358 u16 tss_selector, u16 old_tss_sel,
2359 ulong old_tss_base, struct desc_struct *new_desc)
2361 struct tss_segment_32 tss_seg;
2363 u32 err, new_tss_base = get_desc_base(new_desc);
2365 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2367 if (ret == X86EMUL_PROPAGATE_FAULT) {
2368 /* FIXME: need to provide precise fault address */
2369 emulate_pf(ctxt, old_tss_base, err);
2373 save_state_to_tss32(ctxt, ops, &tss_seg);
2375 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2377 if (ret == X86EMUL_PROPAGATE_FAULT) {
2378 /* FIXME: need to provide precise fault address */
2379 emulate_pf(ctxt, old_tss_base, err);
2383 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2385 if (ret == X86EMUL_PROPAGATE_FAULT) {
2386 /* FIXME: need to provide precise fault address */
2387 emulate_pf(ctxt, new_tss_base, err);
2391 if (old_tss_sel != 0xffff) {
2392 tss_seg.prev_task_link = old_tss_sel;
2394 ret = ops->write_std(new_tss_base,
2395 &tss_seg.prev_task_link,
2396 sizeof tss_seg.prev_task_link,
2398 if (ret == X86EMUL_PROPAGATE_FAULT) {
2399 /* FIXME: need to provide precise fault address */
2400 emulate_pf(ctxt, new_tss_base, err);
2405 return load_state_from_tss32(ctxt, ops, &tss_seg);
2408 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2409 struct x86_emulate_ops *ops,
2410 u16 tss_selector, int reason,
2411 bool has_error_code, u32 error_code)
2413 struct desc_struct curr_tss_desc, next_tss_desc;
2415 u16 old_tss_sel = ops->get_segment_selector(VCPU_SREG_TR, ctxt->vcpu);
2416 ulong old_tss_base =
2417 ops->get_cached_segment_base(VCPU_SREG_TR, ctxt->vcpu);
2420 /* FIXME: old_tss_base == ~0 ? */
2422 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2423 if (ret != X86EMUL_CONTINUE)
2425 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2426 if (ret != X86EMUL_CONTINUE)
2429 /* FIXME: check that next_tss_desc is tss */
2431 if (reason != TASK_SWITCH_IRET) {
2432 if ((tss_selector & 3) > next_tss_desc.dpl ||
2433 ops->cpl(ctxt->vcpu) > next_tss_desc.dpl) {
2434 emulate_gp(ctxt, 0);
2435 return X86EMUL_PROPAGATE_FAULT;
2439 desc_limit = desc_limit_scaled(&next_tss_desc);
2440 if (!next_tss_desc.p ||
2441 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2442 desc_limit < 0x2b)) {
2443 emulate_ts(ctxt, tss_selector & 0xfffc);
2444 return X86EMUL_PROPAGATE_FAULT;
2447 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2448 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2449 write_segment_descriptor(ctxt, ops, old_tss_sel,
2453 if (reason == TASK_SWITCH_IRET)
2454 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2456 /* set back link to prev task only if NT bit is set in eflags
2457 note that old_tss_sel is not used afetr this point */
2458 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2459 old_tss_sel = 0xffff;
2461 if (next_tss_desc.type & 8)
2462 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2463 old_tss_base, &next_tss_desc);
2465 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2466 old_tss_base, &next_tss_desc);
2467 if (ret != X86EMUL_CONTINUE)
2470 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2471 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2473 if (reason != TASK_SWITCH_IRET) {
2474 next_tss_desc.type |= (1 << 1); /* set busy flag */
2475 write_segment_descriptor(ctxt, ops, tss_selector,
2479 ops->set_cr(0, ops->get_cr(0, ctxt->vcpu) | X86_CR0_TS, ctxt->vcpu);
2480 ops->set_cached_descriptor(&next_tss_desc, VCPU_SREG_TR, ctxt->vcpu);
2481 ops->set_segment_selector(tss_selector, VCPU_SREG_TR, ctxt->vcpu);
2483 if (has_error_code) {
2484 struct decode_cache *c = &ctxt->decode;
2486 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2488 c->src.val = (unsigned long) error_code;
2489 emulate_push(ctxt, ops);
2495 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2496 struct x86_emulate_ops *ops,
2497 u16 tss_selector, int reason,
2498 bool has_error_code, u32 error_code)
2500 struct decode_cache *c = &ctxt->decode;
2504 c->dst.type = OP_NONE;
2506 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2507 has_error_code, error_code);
2509 if (rc == X86EMUL_CONTINUE) {
2510 rc = writeback(ctxt, ops);
2511 if (rc == X86EMUL_CONTINUE)
2515 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
2518 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned long base,
2519 int reg, struct operand *op)
2521 struct decode_cache *c = &ctxt->decode;
2522 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2524 register_address_increment(c, &c->regs[reg], df * op->bytes);
2525 op->ptr = (unsigned long *)register_address(c, base, c->regs[reg]);
2529 x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2532 struct decode_cache *c = &ctxt->decode;
2533 int rc = X86EMUL_CONTINUE;
2534 int saved_dst_type = c->dst.type;
2536 ctxt->decode.mem_read.pos = 0;
2538 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
2543 /* LOCK prefix is allowed only with some instructions */
2544 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
2549 /* Privileged instruction can be executed only in CPL=0 */
2550 if ((c->d & Priv) && ops->cpl(ctxt->vcpu)) {
2551 emulate_gp(ctxt, 0);
2555 if (c->rep_prefix && (c->d & String)) {
2556 ctxt->restart = true;
2557 /* All REP prefixes have the same first termination condition */
2558 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
2560 ctxt->restart = false;
2564 /* The second termination condition only applies for REPE
2565 * and REPNE. Test if the repeat string operation prefix is
2566 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
2567 * corresponding termination condition according to:
2568 * - if REPE/REPZ and ZF = 0 then done
2569 * - if REPNE/REPNZ and ZF = 1 then done
2571 if ((c->b == 0xa6) || (c->b == 0xa7) ||
2572 (c->b == 0xae) || (c->b == 0xaf)) {
2573 if ((c->rep_prefix == REPE_PREFIX) &&
2574 ((ctxt->eflags & EFLG_ZF) == 0))
2576 if ((c->rep_prefix == REPNE_PREFIX) &&
2577 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))
2583 if (c->src.type == OP_MEM) {
2584 rc = read_emulated(ctxt, ops, (unsigned long)c->src.ptr,
2585 c->src.valptr, c->src.bytes);
2586 if (rc != X86EMUL_CONTINUE)
2588 c->src.orig_val = c->src.val;
2591 if (c->src2.type == OP_MEM) {
2592 rc = read_emulated(ctxt, ops, (unsigned long)c->src2.ptr,
2593 &c->src2.val, c->src2.bytes);
2594 if (rc != X86EMUL_CONTINUE)
2598 if ((c->d & DstMask) == ImplicitOps)
2602 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
2603 /* optimisation - avoid slow emulated read if Mov */
2604 rc = read_emulated(ctxt, ops, (unsigned long)c->dst.ptr,
2605 &c->dst.val, c->dst.bytes);
2606 if (rc != X86EMUL_CONTINUE)
2609 c->dst.orig_val = c->dst.val;
2619 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
2621 case 0x06: /* push es */
2622 emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
2624 case 0x07: /* pop es */
2625 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
2626 if (rc != X86EMUL_CONTINUE)
2631 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2633 case 0x0e: /* push cs */
2634 emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
2638 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
2640 case 0x16: /* push ss */
2641 emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
2643 case 0x17: /* pop ss */
2644 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
2645 if (rc != X86EMUL_CONTINUE)
2650 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
2652 case 0x1e: /* push ds */
2653 emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
2655 case 0x1f: /* pop ds */
2656 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
2657 if (rc != X86EMUL_CONTINUE)
2662 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
2666 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
2670 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
2674 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2676 case 0x40 ... 0x47: /* inc r16/r32 */
2677 emulate_1op("inc", c->dst, ctxt->eflags);
2679 case 0x48 ... 0x4f: /* dec r16/r32 */
2680 emulate_1op("dec", c->dst, ctxt->eflags);
2682 case 0x50 ... 0x57: /* push reg */
2683 emulate_push(ctxt, ops);
2685 case 0x58 ... 0x5f: /* pop reg */
2687 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
2688 if (rc != X86EMUL_CONTINUE)
2691 case 0x60: /* pusha */
2692 emulate_pusha(ctxt, ops);
2694 case 0x61: /* popa */
2695 rc = emulate_popa(ctxt, ops);
2696 if (rc != X86EMUL_CONTINUE)
2699 case 0x63: /* movsxd */
2700 if (ctxt->mode != X86EMUL_MODE_PROT64)
2701 goto cannot_emulate;
2702 c->dst.val = (s32) c->src.val;
2704 case 0x68: /* push imm */
2705 case 0x6a: /* push imm8 */
2706 emulate_push(ctxt, ops);
2708 case 0x6c: /* insb */
2709 case 0x6d: /* insw/insd */
2710 c->dst.bytes = min(c->dst.bytes, 4u);
2711 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2713 emulate_gp(ctxt, 0);
2716 if (!pio_in_emulated(ctxt, ops, c->dst.bytes,
2717 c->regs[VCPU_REGS_RDX], &c->dst.val))
2718 goto done; /* IO is needed, skip writeback */
2720 case 0x6e: /* outsb */
2721 case 0x6f: /* outsw/outsd */
2722 c->src.bytes = min(c->src.bytes, 4u);
2723 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2725 emulate_gp(ctxt, 0);
2728 ops->pio_out_emulated(c->src.bytes, c->regs[VCPU_REGS_RDX],
2729 &c->src.val, 1, ctxt->vcpu);
2731 c->dst.type = OP_NONE; /* nothing to writeback */
2733 case 0x70 ... 0x7f: /* jcc (short) */
2734 if (test_cc(c->b, ctxt->eflags))
2735 jmp_rel(c, c->src.val);
2737 case 0x80 ... 0x83: /* Grp1 */
2738 switch (c->modrm_reg) {
2759 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
2761 case 0x86 ... 0x87: /* xchg */
2763 /* Write back the register source. */
2764 switch (c->dst.bytes) {
2766 *(u8 *) c->src.ptr = (u8) c->dst.val;
2769 *(u16 *) c->src.ptr = (u16) c->dst.val;
2772 *c->src.ptr = (u32) c->dst.val;
2773 break; /* 64b reg: zero-extend */
2775 *c->src.ptr = c->dst.val;
2779 * Write back the memory destination with implicit LOCK
2782 c->dst.val = c->src.val;
2785 case 0x88 ... 0x8b: /* mov */
2787 case 0x8c: /* mov r/m, sreg */
2788 if (c->modrm_reg > VCPU_SREG_GS) {
2792 c->dst.val = ops->get_segment_selector(c->modrm_reg, ctxt->vcpu);
2794 case 0x8d: /* lea r16/r32, m */
2795 c->dst.val = c->modrm_ea;
2797 case 0x8e: { /* mov seg, r/m16 */
2802 if (c->modrm_reg == VCPU_SREG_CS ||
2803 c->modrm_reg > VCPU_SREG_GS) {
2808 if (c->modrm_reg == VCPU_SREG_SS)
2809 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2811 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
2813 c->dst.type = OP_NONE; /* Disable writeback. */
2816 case 0x8f: /* pop (sole member of Grp1a) */
2817 rc = emulate_grp1a(ctxt, ops);
2818 if (rc != X86EMUL_CONTINUE)
2821 case 0x90: /* nop / xchg r8,rax */
2822 if (c->dst.ptr == (unsigned long *)&c->regs[VCPU_REGS_RAX]) {
2823 c->dst.type = OP_NONE; /* nop */
2826 case 0x91 ... 0x97: /* xchg reg,rax */
2827 c->src.type = OP_REG;
2828 c->src.bytes = c->op_bytes;
2829 c->src.ptr = (unsigned long *) &c->regs[VCPU_REGS_RAX];
2830 c->src.val = *(c->src.ptr);
2832 case 0x9c: /* pushf */
2833 c->src.val = (unsigned long) ctxt->eflags;
2834 emulate_push(ctxt, ops);
2836 case 0x9d: /* popf */
2837 c->dst.type = OP_REG;
2838 c->dst.ptr = (unsigned long *) &ctxt->eflags;
2839 c->dst.bytes = c->op_bytes;
2840 rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
2841 if (rc != X86EMUL_CONTINUE)
2844 case 0xa0 ... 0xa1: /* mov */
2845 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2846 c->dst.val = c->src.val;
2848 case 0xa2 ... 0xa3: /* mov */
2849 c->dst.val = (unsigned long)c->regs[VCPU_REGS_RAX];
2851 case 0xa4 ... 0xa5: /* movs */
2853 case 0xa6 ... 0xa7: /* cmps */
2854 c->dst.type = OP_NONE; /* Disable writeback. */
2855 DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.ptr, c->dst.ptr);
2857 case 0xa8 ... 0xa9: /* test ax, imm */
2859 case 0xaa ... 0xab: /* stos */
2860 c->dst.val = c->regs[VCPU_REGS_RAX];
2862 case 0xac ... 0xad: /* lods */
2864 case 0xae ... 0xaf: /* scas */
2865 DPRINTF("Urk! I don't handle SCAS.\n");
2866 goto cannot_emulate;
2867 case 0xb0 ... 0xbf: /* mov r, imm */
2872 case 0xc3: /* ret */
2873 c->dst.type = OP_REG;
2874 c->dst.ptr = &c->eip;
2875 c->dst.bytes = c->op_bytes;
2876 goto pop_instruction;
2877 case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */
2879 c->dst.val = c->src.val;
2881 case 0xcb: /* ret far */
2882 rc = emulate_ret_far(ctxt, ops);
2883 if (rc != X86EMUL_CONTINUE)
2886 case 0xd0 ... 0xd1: /* Grp2 */
2890 case 0xd2 ... 0xd3: /* Grp2 */
2891 c->src.val = c->regs[VCPU_REGS_RCX];
2894 case 0xe4: /* inb */
2897 case 0xe6: /* outb */
2898 case 0xe7: /* out */
2900 case 0xe8: /* call (near) */ {
2901 long int rel = c->src.val;
2902 c->src.val = (unsigned long) c->eip;
2904 emulate_push(ctxt, ops);
2907 case 0xe9: /* jmp rel */
2909 case 0xea: { /* jmp far */
2912 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
2914 if (load_segment_descriptor(ctxt, ops, sel, VCPU_SREG_CS))
2918 memcpy(&c->eip, c->src.valptr, c->op_bytes);
2922 jmp: /* jmp rel short */
2923 jmp_rel(c, c->src.val);
2924 c->dst.type = OP_NONE; /* Disable writeback. */
2926 case 0xec: /* in al,dx */
2927 case 0xed: /* in (e/r)ax,dx */
2928 c->src.val = c->regs[VCPU_REGS_RDX];
2930 c->dst.bytes = min(c->dst.bytes, 4u);
2931 if (!emulator_io_permited(ctxt, ops, c->src.val, c->dst.bytes)) {
2932 emulate_gp(ctxt, 0);
2935 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
2937 goto done; /* IO is needed */
2939 case 0xee: /* out al,dx */
2940 case 0xef: /* out (e/r)ax,dx */
2941 c->src.val = c->regs[VCPU_REGS_RDX];
2943 c->dst.bytes = min(c->dst.bytes, 4u);
2944 if (!emulator_io_permited(ctxt, ops, c->src.val, c->dst.bytes)) {
2945 emulate_gp(ctxt, 0);
2948 ops->pio_out_emulated(c->dst.bytes, c->src.val, &c->dst.val, 1,
2950 c->dst.type = OP_NONE; /* Disable writeback. */
2952 case 0xf4: /* hlt */
2953 ctxt->vcpu->arch.halt_request = 1;
2955 case 0xf5: /* cmc */
2956 /* complement carry flag from eflags reg */
2957 ctxt->eflags ^= EFLG_CF;
2958 c->dst.type = OP_NONE; /* Disable writeback. */
2960 case 0xf6 ... 0xf7: /* Grp3 */
2961 if (!emulate_grp3(ctxt, ops))
2962 goto cannot_emulate;
2964 case 0xf8: /* clc */
2965 ctxt->eflags &= ~EFLG_CF;
2966 c->dst.type = OP_NONE; /* Disable writeback. */
2968 case 0xfa: /* cli */
2969 if (emulator_bad_iopl(ctxt, ops))
2970 emulate_gp(ctxt, 0);
2972 ctxt->eflags &= ~X86_EFLAGS_IF;
2973 c->dst.type = OP_NONE; /* Disable writeback. */
2976 case 0xfb: /* sti */
2977 if (emulator_bad_iopl(ctxt, ops))
2978 emulate_gp(ctxt, 0);
2980 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
2981 ctxt->eflags |= X86_EFLAGS_IF;
2982 c->dst.type = OP_NONE; /* Disable writeback. */
2985 case 0xfc: /* cld */
2986 ctxt->eflags &= ~EFLG_DF;
2987 c->dst.type = OP_NONE; /* Disable writeback. */
2989 case 0xfd: /* std */
2990 ctxt->eflags |= EFLG_DF;
2991 c->dst.type = OP_NONE; /* Disable writeback. */
2993 case 0xfe: /* Grp4 */
2995 rc = emulate_grp45(ctxt, ops);
2996 if (rc != X86EMUL_CONTINUE)
2999 case 0xff: /* Grp5 */
3000 if (c->modrm_reg == 5)
3006 rc = writeback(ctxt, ops);
3007 if (rc != X86EMUL_CONTINUE)
3011 * restore dst type in case the decoding will be reused
3012 * (happens for string instruction )
3014 c->dst.type = saved_dst_type;
3016 if ((c->d & SrcMask) == SrcSI)
3017 string_addr_inc(ctxt, seg_override_base(ctxt, ops, c),
3018 VCPU_REGS_RSI, &c->src);
3020 if ((c->d & DstMask) == DstDI)
3021 string_addr_inc(ctxt, es_base(ctxt, ops), VCPU_REGS_RDI,
3024 if (c->rep_prefix && (c->d & String)) {
3025 struct read_cache *rc = &ctxt->decode.io_read;
3026 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3028 * Re-enter guest when pio read ahead buffer is empty or,
3029 * if it is not used, after each 1024 iteration.
3031 if ((rc->end == 0 && !(c->regs[VCPU_REGS_RCX] & 0x3ff)) ||
3032 (rc->end != 0 && rc->end == rc->pos))
3033 ctxt->restart = false;
3036 * reset read cache here in case string instruction is restared
3039 ctxt->decode.mem_read.end = 0;
3043 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
3047 case 0x01: /* lgdt, lidt, lmsw */
3048 switch (c->modrm_reg) {
3050 unsigned long address;
3052 case 0: /* vmcall */
3053 if (c->modrm_mod != 3 || c->modrm_rm != 1)
3054 goto cannot_emulate;
3056 rc = kvm_fix_hypercall(ctxt->vcpu);
3057 if (rc != X86EMUL_CONTINUE)
3060 /* Let the processor re-execute the fixed hypercall */
3062 /* Disable writeback. */
3063 c->dst.type = OP_NONE;
3066 rc = read_descriptor(ctxt, ops, c->src.ptr,
3067 &size, &address, c->op_bytes);
3068 if (rc != X86EMUL_CONTINUE)
3070 realmode_lgdt(ctxt->vcpu, size, address);
3071 /* Disable writeback. */
3072 c->dst.type = OP_NONE;
3074 case 3: /* lidt/vmmcall */
3075 if (c->modrm_mod == 3) {
3076 switch (c->modrm_rm) {
3078 rc = kvm_fix_hypercall(ctxt->vcpu);
3079 if (rc != X86EMUL_CONTINUE)
3083 goto cannot_emulate;
3086 rc = read_descriptor(ctxt, ops, c->src.ptr,
3089 if (rc != X86EMUL_CONTINUE)
3091 realmode_lidt(ctxt->vcpu, size, address);
3093 /* Disable writeback. */
3094 c->dst.type = OP_NONE;
3098 c->dst.val = ops->get_cr(0, ctxt->vcpu);
3101 ops->set_cr(0, (ops->get_cr(0, ctxt->vcpu) & ~0x0ful) |
3102 (c->src.val & 0x0f), ctxt->vcpu);
3103 c->dst.type = OP_NONE;
3105 case 5: /* not defined */
3109 emulate_invlpg(ctxt->vcpu, c->modrm_ea);
3110 /* Disable writeback. */
3111 c->dst.type = OP_NONE;
3114 goto cannot_emulate;
3117 case 0x05: /* syscall */
3118 rc = emulate_syscall(ctxt, ops);
3119 if (rc != X86EMUL_CONTINUE)
3125 emulate_clts(ctxt->vcpu);
3126 c->dst.type = OP_NONE;
3128 case 0x08: /* invd */
3129 case 0x09: /* wbinvd */
3130 case 0x0d: /* GrpP (prefetch) */
3131 case 0x18: /* Grp16 (prefetch/nop) */
3132 c->dst.type = OP_NONE;
3134 case 0x20: /* mov cr, reg */
3135 switch (c->modrm_reg) {
3142 c->regs[c->modrm_rm] = ops->get_cr(c->modrm_reg, ctxt->vcpu);
3143 c->dst.type = OP_NONE; /* no writeback */
3145 case 0x21: /* mov from dr to reg */
3146 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
3147 (c->modrm_reg == 4 || c->modrm_reg == 5)) {
3151 ops->get_dr(c->modrm_reg, &c->regs[c->modrm_rm], ctxt->vcpu);
3152 c->dst.type = OP_NONE; /* no writeback */
3154 case 0x22: /* mov reg, cr */
3155 if (ops->set_cr(c->modrm_reg, c->modrm_val, ctxt->vcpu)) {
3156 emulate_gp(ctxt, 0);
3159 c->dst.type = OP_NONE;
3161 case 0x23: /* mov from reg to dr */
3162 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
3163 (c->modrm_reg == 4 || c->modrm_reg == 5)) {
3168 if (ops->set_dr(c->modrm_reg, c->regs[c->modrm_rm] &
3169 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
3170 ~0ULL : ~0U), ctxt->vcpu) < 0) {
3171 /* #UD condition is already handled by the code above */
3172 emulate_gp(ctxt, 0);
3176 c->dst.type = OP_NONE; /* no writeback */
3180 msr_data = (u32)c->regs[VCPU_REGS_RAX]
3181 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
3182 if (ops->set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) {
3183 emulate_gp(ctxt, 0);
3186 rc = X86EMUL_CONTINUE;
3187 c->dst.type = OP_NONE;
3191 if (ops->get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) {
3192 emulate_gp(ctxt, 0);
3195 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
3196 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
3198 rc = X86EMUL_CONTINUE;
3199 c->dst.type = OP_NONE;
3201 case 0x34: /* sysenter */
3202 rc = emulate_sysenter(ctxt, ops);
3203 if (rc != X86EMUL_CONTINUE)
3208 case 0x35: /* sysexit */
3209 rc = emulate_sysexit(ctxt, ops);
3210 if (rc != X86EMUL_CONTINUE)
3215 case 0x40 ... 0x4f: /* cmov */
3216 c->dst.val = c->dst.orig_val = c->src.val;
3217 if (!test_cc(c->b, ctxt->eflags))
3218 c->dst.type = OP_NONE; /* no writeback */
3220 case 0x80 ... 0x8f: /* jnz rel, etc*/
3221 if (test_cc(c->b, ctxt->eflags))
3222 jmp_rel(c, c->src.val);
3223 c->dst.type = OP_NONE;
3225 case 0xa0: /* push fs */
3226 emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
3228 case 0xa1: /* pop fs */
3229 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
3230 if (rc != X86EMUL_CONTINUE)
3235 c->dst.type = OP_NONE;
3236 /* only subword offset */
3237 c->src.val &= (c->dst.bytes << 3) - 1;
3238 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
3240 case 0xa4: /* shld imm8, r, r/m */
3241 case 0xa5: /* shld cl, r, r/m */
3242 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
3244 case 0xa8: /* push gs */
3245 emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
3247 case 0xa9: /* pop gs */
3248 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
3249 if (rc != X86EMUL_CONTINUE)
3254 /* only subword offset */
3255 c->src.val &= (c->dst.bytes << 3) - 1;
3256 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
3258 case 0xac: /* shrd imm8, r, r/m */
3259 case 0xad: /* shrd cl, r, r/m */
3260 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
3262 case 0xae: /* clflush */
3264 case 0xb0 ... 0xb1: /* cmpxchg */
3266 * Save real source value, then compare EAX against
3269 c->src.orig_val = c->src.val;
3270 c->src.val = c->regs[VCPU_REGS_RAX];
3271 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
3272 if (ctxt->eflags & EFLG_ZF) {
3273 /* Success: write back to memory. */
3274 c->dst.val = c->src.orig_val;
3276 /* Failure: write the value we saw to EAX. */
3277 c->dst.type = OP_REG;
3278 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
3283 /* only subword offset */
3284 c->src.val &= (c->dst.bytes << 3) - 1;
3285 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
3287 case 0xb6 ... 0xb7: /* movzx */
3288 c->dst.bytes = c->op_bytes;
3289 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
3292 case 0xba: /* Grp8 */
3293 switch (c->modrm_reg & 3) {
3306 /* only subword offset */
3307 c->src.val &= (c->dst.bytes << 3) - 1;
3308 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
3310 case 0xbe ... 0xbf: /* movsx */
3311 c->dst.bytes = c->op_bytes;
3312 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
3315 case 0xc3: /* movnti */
3316 c->dst.bytes = c->op_bytes;
3317 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
3320 case 0xc7: /* Grp9 (cmpxchg8b) */
3321 rc = emulate_grp9(ctxt, ops);
3322 if (rc != X86EMUL_CONTINUE)
3329 DPRINTF("Cannot emulate %02x\n", c->b);