1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affilates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
26 #include <public/xen.h>
27 #define DPRINTF(_f, _a ...) printf(_f , ## _a)
29 #include <linux/kvm_host.h>
30 #include "kvm_cache_regs.h"
31 #define DPRINTF(x...) do {} while (0)
33 #include <linux/module.h>
34 #include <asm/kvm_emulate.h>
40 * Opcode effective-address decode tables.
41 * Note that we only emulate instructions that have at least one memory
42 * operand (excluding implicit stack references). We assume that stack
43 * references and instruction fetches will never occur in special memory
44 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
48 /* Operand sizes: 8-bit operands or specified/overridden size. */
49 #define ByteOp (1<<0) /* 8-bit operands. */
50 /* Destination operand type. */
51 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
52 #define DstReg (2<<1) /* Register operand. */
53 #define DstMem (3<<1) /* Memory operand. */
54 #define DstAcc (4<<1) /* Destination Accumulator */
55 #define DstDI (5<<1) /* Destination is in ES:(E)DI */
56 #define DstMem64 (6<<1) /* 64bit memory operand */
57 #define DstMask (7<<1)
58 /* Source operand type. */
59 #define SrcNone (0<<4) /* No source operand. */
60 #define SrcImplicit (0<<4) /* Source operand is implicit in the opcode. */
61 #define SrcReg (1<<4) /* Register operand. */
62 #define SrcMem (2<<4) /* Memory operand. */
63 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
64 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
65 #define SrcImm (5<<4) /* Immediate operand. */
66 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
67 #define SrcOne (7<<4) /* Implied '1' */
68 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
69 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
70 #define SrcSI (0xa<<4) /* Source is in the DS:RSI */
71 #define SrcImmFAddr (0xb<<4) /* Source is immediate far address */
72 #define SrcMemFAddr (0xc<<4) /* Source is far address in memory */
73 #define SrcAcc (0xd<<4) /* Source Accumulator */
74 #define SrcMask (0xf<<4)
75 /* Generic ModRM decode. */
77 /* Destination is only written; never read. */
80 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
81 #define String (1<<12) /* String instruction (rep capable) */
82 #define Stack (1<<13) /* Stack instruction (push/pop) */
83 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
84 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
85 #define GroupMask 0xff /* Group number stored in bits 0:7 */
87 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
88 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
90 /* Source 2 operand type */
91 #define Src2None (0<<29)
92 #define Src2CL (1<<29)
93 #define Src2ImmByte (2<<29)
94 #define Src2One (3<<29)
95 #define Src2Mask (7<<29)
97 #define X2(x) (x), (x)
98 #define X3(x) X2(x), (x)
99 #define X4(x) X2(x), X2(x)
100 #define X5(x) X4(x), (x)
101 #define X6(x) X4(x), X2(x)
102 #define X7(x) X4(x), X3(x)
103 #define X8(x) X4(x), X4(x)
104 #define X16(x) X8(x), X8(x)
107 Group1_80, Group1_81, Group1_82, Group1_83,
108 Group1A, Group3_Byte, Group3, Group4, Group5, Group7,
112 static u32 opcode_table[256] = {
114 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
115 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
116 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
117 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
119 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
120 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
121 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
122 ImplicitOps | Stack | No64, 0,
124 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
125 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
126 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
127 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
129 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
130 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
131 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
132 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
134 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
135 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
136 ByteOp | DstAcc | SrcImmByte, DstAcc | SrcImm, 0, 0,
138 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
139 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
140 ByteOp | DstAcc | SrcImmByte, DstAcc | SrcImm, 0, 0,
142 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
143 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
144 ByteOp | DstAcc | SrcImmByte, DstAcc | SrcImm, 0, 0,
146 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
147 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
148 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
157 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
158 0, DstReg | SrcMem32 | ModRM | Mov /* movsxd (x86/64) */ ,
161 SrcImm | Mov | Stack, 0, SrcImmByte | Mov | Stack, 0,
162 DstDI | ByteOp | Mov | String, DstDI | Mov | String, /* insb, insw/insd */
163 SrcSI | ByteOp | ImplicitOps | String, SrcSI | ImplicitOps | String, /* outsb, outsw/outsd */
167 Group | Group1_80, Group | Group1_81,
168 Group | Group1_82, Group | Group1_83,
169 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
170 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
172 ByteOp | DstMem | SrcReg | ModRM | Mov, DstMem | SrcReg | ModRM | Mov,
173 ByteOp | DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
174 DstMem | SrcNone | ModRM | Mov, ModRM | DstReg,
175 ImplicitOps | SrcMem16 | ModRM, Group | Group1A,
177 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
179 0, 0, SrcImmFAddr | No64, 0,
180 ImplicitOps | Stack, ImplicitOps | Stack, 0, 0,
182 ByteOp | DstAcc | SrcMem | Mov | MemAbs, DstAcc | SrcMem | Mov | MemAbs,
183 ByteOp | DstMem | SrcAcc | Mov | MemAbs, DstMem | SrcAcc | Mov | MemAbs,
184 ByteOp | SrcSI | DstDI | Mov | String, SrcSI | DstDI | Mov | String,
185 ByteOp | SrcSI | DstDI | String, SrcSI | DstDI | String,
187 DstAcc | SrcImmByte | ByteOp, DstAcc | SrcImm, ByteOp | DstDI | Mov | String, DstDI | Mov | String,
188 ByteOp | SrcSI | DstAcc | Mov | String, SrcSI | DstAcc | Mov | String,
189 ByteOp | DstDI | String, DstDI | String,
191 X8(ByteOp | DstReg | SrcImm | Mov),
193 X8(DstReg | SrcImm | Mov),
195 ByteOp | DstMem | SrcImm | ModRM, DstMem | SrcImmByte | ModRM,
196 0, ImplicitOps | Stack, 0, 0,
197 ByteOp | DstMem | SrcImm | ModRM | Mov, DstMem | SrcImm | ModRM | Mov,
199 0, 0, 0, ImplicitOps | Stack,
200 ImplicitOps, SrcImmByte, ImplicitOps | No64, ImplicitOps,
202 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
203 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
206 0, 0, 0, 0, 0, 0, 0, 0,
209 ByteOp | SrcImmUByte | DstAcc, SrcImmUByte | DstAcc,
210 ByteOp | SrcImmUByte | DstAcc, SrcImmUByte | DstAcc,
212 SrcImm | Stack, SrcImm | ImplicitOps,
213 SrcImmFAddr | No64, SrcImmByte | ImplicitOps,
214 SrcNone | ByteOp | DstAcc, SrcNone | DstAcc,
215 SrcNone | ByteOp | DstAcc, SrcNone | DstAcc,
218 ImplicitOps | Priv, ImplicitOps, Group | Group3_Byte, Group | Group3,
220 ImplicitOps, 0, ImplicitOps, ImplicitOps,
221 ImplicitOps, ImplicitOps, Group | Group4, Group | Group5,
224 static u32 twobyte_table[256] = {
226 0, Group | GroupDual | Group7, 0, 0,
227 0, ImplicitOps, ImplicitOps | Priv, 0,
228 ImplicitOps | Priv, ImplicitOps | Priv, 0, 0,
229 0, ImplicitOps | ModRM, 0, 0,
231 0, 0, 0, 0, 0, 0, 0, 0, ImplicitOps | ModRM, 0, 0, 0, 0, 0, 0, 0,
233 ModRM | ImplicitOps | Priv, ModRM | Priv,
234 ModRM | ImplicitOps | Priv, ModRM | Priv,
236 0, 0, 0, 0, 0, 0, 0, 0,
238 ImplicitOps | Priv, 0, ImplicitOps | Priv, 0,
239 ImplicitOps, ImplicitOps | Priv, 0, 0,
240 0, 0, 0, 0, 0, 0, 0, 0,
242 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
243 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
244 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
245 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
247 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
248 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
249 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
250 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
252 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
254 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
256 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
258 SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm,
259 SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm,
261 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
263 ImplicitOps | Stack, ImplicitOps | Stack,
264 0, DstMem | SrcReg | ModRM | BitOp,
265 DstMem | SrcReg | Src2ImmByte | ModRM,
266 DstMem | SrcReg | Src2CL | ModRM, 0, 0,
268 ImplicitOps | Stack, ImplicitOps | Stack,
269 0, DstMem | SrcReg | ModRM | BitOp | Lock,
270 DstMem | SrcReg | Src2ImmByte | ModRM,
271 DstMem | SrcReg | Src2CL | ModRM,
274 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
275 0, DstMem | SrcReg | ModRM | BitOp | Lock,
276 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
277 DstReg | SrcMem16 | ModRM | Mov,
280 Group | Group8, DstMem | SrcReg | ModRM | BitOp | Lock,
281 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
282 DstReg | SrcMem16 | ModRM | Mov,
284 0, 0, 0, DstMem | SrcReg | ModRM | Mov,
285 0, 0, 0, Group | GroupDual | Group9,
286 0, 0, 0, 0, 0, 0, 0, 0,
288 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
290 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
292 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
295 static u32 group_table[] = {
297 ByteOp | DstMem | SrcImm | ModRM | Lock,
298 ByteOp | DstMem | SrcImm | ModRM | Lock,
299 ByteOp | DstMem | SrcImm | ModRM | Lock,
300 ByteOp | DstMem | SrcImm | ModRM | Lock,
301 ByteOp | DstMem | SrcImm | ModRM | Lock,
302 ByteOp | DstMem | SrcImm | ModRM | Lock,
303 ByteOp | DstMem | SrcImm | ModRM | Lock,
304 ByteOp | DstMem | SrcImm | ModRM,
306 DstMem | SrcImm | ModRM | Lock,
307 DstMem | SrcImm | ModRM | Lock,
308 DstMem | SrcImm | ModRM | Lock,
309 DstMem | SrcImm | ModRM | Lock,
310 DstMem | SrcImm | ModRM | Lock,
311 DstMem | SrcImm | ModRM | Lock,
312 DstMem | SrcImm | ModRM | Lock,
313 DstMem | SrcImm | ModRM,
315 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
316 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
317 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
318 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
319 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
320 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
321 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
322 ByteOp | DstMem | SrcImm | ModRM | No64,
324 DstMem | SrcImmByte | ModRM | Lock,
325 DstMem | SrcImmByte | ModRM | Lock,
326 DstMem | SrcImmByte | ModRM | Lock,
327 DstMem | SrcImmByte | ModRM | Lock,
328 DstMem | SrcImmByte | ModRM | Lock,
329 DstMem | SrcImmByte | ModRM | Lock,
330 DstMem | SrcImmByte | ModRM | Lock,
331 DstMem | SrcImmByte | ModRM,
333 DstMem | SrcNone | ModRM | Mov | Stack, 0, 0, 0, 0, 0, 0, 0,
335 ByteOp | SrcImm | DstMem | ModRM, ByteOp | SrcImm | DstMem | ModRM,
336 ByteOp | DstMem | SrcNone | ModRM, ByteOp | DstMem | SrcNone | ModRM,
339 DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
340 DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM,
343 ByteOp | DstMem | SrcNone | ModRM | Lock, ByteOp | DstMem | SrcNone | ModRM | Lock,
346 DstMem | SrcNone | ModRM | Lock, DstMem | SrcNone | ModRM | Lock,
347 SrcMem | ModRM | Stack, 0,
348 SrcMem | ModRM | Stack, SrcMemFAddr | ModRM | ImplicitOps,
349 SrcMem | ModRM | Stack, 0,
351 0, 0, ModRM | SrcMem | Priv, ModRM | SrcMem | Priv,
352 SrcNone | ModRM | DstMem | Mov, 0,
353 SrcMem16 | ModRM | Mov | Priv, SrcMem | ModRM | ByteOp | Priv,
356 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM | Lock,
357 DstMem | SrcImmByte | ModRM | Lock, DstMem | SrcImmByte | ModRM | Lock,
359 0, DstMem64 | ModRM | Lock, 0, 0, 0, 0, 0, 0,
362 static u32 group2_table[] = {
364 SrcNone | ModRM | Priv, 0, 0, SrcNone | ModRM | Priv,
365 SrcNone | ModRM | DstMem | Mov, 0,
366 SrcMem16 | ModRM | Mov | Priv, 0,
368 0, 0, 0, 0, 0, 0, 0, 0,
371 /* EFLAGS bit definitions. */
372 #define EFLG_ID (1<<21)
373 #define EFLG_VIP (1<<20)
374 #define EFLG_VIF (1<<19)
375 #define EFLG_AC (1<<18)
376 #define EFLG_VM (1<<17)
377 #define EFLG_RF (1<<16)
378 #define EFLG_IOPL (3<<12)
379 #define EFLG_NT (1<<14)
380 #define EFLG_OF (1<<11)
381 #define EFLG_DF (1<<10)
382 #define EFLG_IF (1<<9)
383 #define EFLG_TF (1<<8)
384 #define EFLG_SF (1<<7)
385 #define EFLG_ZF (1<<6)
386 #define EFLG_AF (1<<4)
387 #define EFLG_PF (1<<2)
388 #define EFLG_CF (1<<0)
391 * Instruction emulation:
392 * Most instructions are emulated directly via a fragment of inline assembly
393 * code. This allows us to save/restore EFLAGS and thus very easily pick up
394 * any modified flags.
397 #if defined(CONFIG_X86_64)
398 #define _LO32 "k" /* force 32-bit operand */
399 #define _STK "%%rsp" /* stack pointer */
400 #elif defined(__i386__)
401 #define _LO32 "" /* force 32-bit operand */
402 #define _STK "%%esp" /* stack pointer */
406 * These EFLAGS bits are restored from saved value during emulation, and
407 * any changes are written back to the saved value after emulation.
409 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
411 /* Before executing instruction: restore necessary bits in EFLAGS. */
412 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
413 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
414 "movl %"_sav",%"_LO32 _tmp"; " \
417 "movl %"_msk",%"_LO32 _tmp"; " \
418 "andl %"_LO32 _tmp",("_STK"); " \
420 "notl %"_LO32 _tmp"; " \
421 "andl %"_LO32 _tmp",("_STK"); " \
422 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
424 "orl %"_LO32 _tmp",("_STK"); " \
428 /* After executing instruction: write-back necessary bits in EFLAGS. */
429 #define _POST_EFLAGS(_sav, _msk, _tmp) \
430 /* _sav |= EFLAGS & _msk; */ \
433 "andl %"_msk",%"_LO32 _tmp"; " \
434 "orl %"_LO32 _tmp",%"_sav"; "
442 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix) \
444 __asm__ __volatile__ ( \
445 _PRE_EFLAGS("0", "4", "2") \
446 _op _suffix " %"_x"3,%1; " \
447 _POST_EFLAGS("0", "4", "2") \
448 : "=m" (_eflags), "=m" ((_dst).val), \
450 : _y ((_src).val), "i" (EFLAGS_MASK)); \
454 /* Raw emulation: instruction has two explicit operands. */
455 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
457 unsigned long _tmp; \
459 switch ((_dst).bytes) { \
461 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w"); \
464 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l"); \
467 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q")); \
472 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
474 unsigned long _tmp; \
475 switch ((_dst).bytes) { \
477 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b"); \
480 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
481 _wx, _wy, _lx, _ly, _qx, _qy); \
486 /* Source operand is byte-sized and may be restricted to just %cl. */
487 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
488 __emulate_2op(_op, _src, _dst, _eflags, \
489 "b", "c", "b", "c", "b", "c", "b", "c")
491 /* Source operand is byte, word, long or quad sized. */
492 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
493 __emulate_2op(_op, _src, _dst, _eflags, \
494 "b", "q", "w", "r", _LO32, "r", "", "r")
496 /* Source operand is word, long or quad sized. */
497 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
498 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
499 "w", "r", _LO32, "r", "", "r")
501 /* Instruction has three operands and one operand is stored in ECX register */
502 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
504 unsigned long _tmp; \
505 _type _clv = (_cl).val; \
506 _type _srcv = (_src).val; \
507 _type _dstv = (_dst).val; \
509 __asm__ __volatile__ ( \
510 _PRE_EFLAGS("0", "5", "2") \
511 _op _suffix " %4,%1 \n" \
512 _POST_EFLAGS("0", "5", "2") \
513 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
514 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
517 (_cl).val = (unsigned long) _clv; \
518 (_src).val = (unsigned long) _srcv; \
519 (_dst).val = (unsigned long) _dstv; \
522 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
524 switch ((_dst).bytes) { \
526 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
527 "w", unsigned short); \
530 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
531 "l", unsigned int); \
534 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
535 "q", unsigned long)); \
540 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
542 unsigned long _tmp; \
544 __asm__ __volatile__ ( \
545 _PRE_EFLAGS("0", "3", "2") \
546 _op _suffix " %1; " \
547 _POST_EFLAGS("0", "3", "2") \
548 : "=m" (_eflags), "+m" ((_dst).val), \
550 : "i" (EFLAGS_MASK)); \
553 /* Instruction has only one explicit operand (no source operand). */
554 #define emulate_1op(_op, _dst, _eflags) \
556 switch ((_dst).bytes) { \
557 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
558 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
559 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
560 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
564 /* Fetch next part of the instruction being emulated. */
565 #define insn_fetch(_type, _size, _eip) \
566 ({ unsigned long _x; \
567 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
568 if (rc != X86EMUL_CONTINUE) \
574 #define insn_fetch_arr(_arr, _size, _eip) \
575 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
576 if (rc != X86EMUL_CONTINUE) \
581 static inline unsigned long ad_mask(struct decode_cache *c)
583 return (1UL << (c->ad_bytes << 3)) - 1;
586 /* Access/update address held in a register, based on addressing mode. */
587 static inline unsigned long
588 address_mask(struct decode_cache *c, unsigned long reg)
590 if (c->ad_bytes == sizeof(unsigned long))
593 return reg & ad_mask(c);
596 static inline unsigned long
597 register_address(struct decode_cache *c, unsigned long base, unsigned long reg)
599 return base + address_mask(c, reg);
603 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
605 if (c->ad_bytes == sizeof(unsigned long))
608 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
611 static inline void jmp_rel(struct decode_cache *c, int rel)
613 register_address_increment(c, &c->eip, rel);
616 static void set_seg_override(struct decode_cache *c, int seg)
618 c->has_seg_override = true;
619 c->seg_override = seg;
622 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
623 struct x86_emulate_ops *ops, int seg)
625 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
628 return ops->get_cached_segment_base(seg, ctxt->vcpu);
631 static unsigned long seg_override_base(struct x86_emulate_ctxt *ctxt,
632 struct x86_emulate_ops *ops,
633 struct decode_cache *c)
635 if (!c->has_seg_override)
638 return seg_base(ctxt, ops, c->seg_override);
641 static unsigned long es_base(struct x86_emulate_ctxt *ctxt,
642 struct x86_emulate_ops *ops)
644 return seg_base(ctxt, ops, VCPU_SREG_ES);
647 static unsigned long ss_base(struct x86_emulate_ctxt *ctxt,
648 struct x86_emulate_ops *ops)
650 return seg_base(ctxt, ops, VCPU_SREG_SS);
653 static void emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
654 u32 error, bool valid)
656 ctxt->exception = vec;
657 ctxt->error_code = error;
658 ctxt->error_code_valid = valid;
659 ctxt->restart = false;
662 static void emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
664 emulate_exception(ctxt, GP_VECTOR, err, true);
667 static void emulate_pf(struct x86_emulate_ctxt *ctxt, unsigned long addr,
671 emulate_exception(ctxt, PF_VECTOR, err, true);
674 static void emulate_ud(struct x86_emulate_ctxt *ctxt)
676 emulate_exception(ctxt, UD_VECTOR, 0, false);
679 static void emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
681 emulate_exception(ctxt, TS_VECTOR, err, true);
684 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
685 struct x86_emulate_ops *ops,
686 unsigned long eip, u8 *dest)
688 struct fetch_cache *fc = &ctxt->decode.fetch;
692 if (eip == fc->end) {
693 cur_size = fc->end - fc->start;
694 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
695 rc = ops->fetch(ctxt->cs_base + eip, fc->data + cur_size,
696 size, ctxt->vcpu, NULL);
697 if (rc != X86EMUL_CONTINUE)
701 *dest = fc->data[eip - fc->start];
702 return X86EMUL_CONTINUE;
705 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
706 struct x86_emulate_ops *ops,
707 unsigned long eip, void *dest, unsigned size)
711 /* x86 instructions are limited to 15 bytes. */
712 if (eip + size - ctxt->eip > 15)
713 return X86EMUL_UNHANDLEABLE;
715 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
716 if (rc != X86EMUL_CONTINUE)
719 return X86EMUL_CONTINUE;
723 * Given the 'reg' portion of a ModRM byte, and a register block, return a
724 * pointer into the block that addresses the relevant register.
725 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
727 static void *decode_register(u8 modrm_reg, unsigned long *regs,
732 p = ®s[modrm_reg];
733 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
734 p = (unsigned char *)®s[modrm_reg & 3] + 1;
738 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
739 struct x86_emulate_ops *ops,
741 u16 *size, unsigned long *address, int op_bytes)
748 rc = ops->read_std((unsigned long)ptr, (unsigned long *)size, 2,
750 if (rc != X86EMUL_CONTINUE)
752 rc = ops->read_std((unsigned long)ptr + 2, address, op_bytes,
757 static int test_cc(unsigned int condition, unsigned int flags)
761 switch ((condition & 15) >> 1) {
763 rc |= (flags & EFLG_OF);
765 case 1: /* b/c/nae */
766 rc |= (flags & EFLG_CF);
769 rc |= (flags & EFLG_ZF);
772 rc |= (flags & (EFLG_CF|EFLG_ZF));
775 rc |= (flags & EFLG_SF);
778 rc |= (flags & EFLG_PF);
781 rc |= (flags & EFLG_ZF);
784 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
788 /* Odd condition identifiers (lsb == 1) have inverted sense. */
789 return (!!rc ^ (condition & 1));
792 static void decode_register_operand(struct operand *op,
793 struct decode_cache *c,
796 unsigned reg = c->modrm_reg;
797 int highbyte_regs = c->rex_prefix == 0;
800 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
802 if ((c->d & ByteOp) && !inhibit_bytereg) {
803 op->ptr = decode_register(reg, c->regs, highbyte_regs);
804 op->val = *(u8 *)op->ptr;
807 op->ptr = decode_register(reg, c->regs, 0);
808 op->bytes = c->op_bytes;
811 op->val = *(u16 *)op->ptr;
814 op->val = *(u32 *)op->ptr;
817 op->val = *(u64 *) op->ptr;
821 op->orig_val = op->val;
824 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
825 struct x86_emulate_ops *ops)
827 struct decode_cache *c = &ctxt->decode;
829 int index_reg = 0, base_reg = 0, scale;
830 int rc = X86EMUL_CONTINUE;
833 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
834 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
835 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
838 c->modrm = insn_fetch(u8, 1, c->eip);
839 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
840 c->modrm_reg |= (c->modrm & 0x38) >> 3;
841 c->modrm_rm |= (c->modrm & 0x07);
845 if (c->modrm_mod == 3) {
846 c->modrm_ptr = decode_register(c->modrm_rm,
847 c->regs, c->d & ByteOp);
848 c->modrm_val = *(unsigned long *)c->modrm_ptr;
852 if (c->ad_bytes == 2) {
853 unsigned bx = c->regs[VCPU_REGS_RBX];
854 unsigned bp = c->regs[VCPU_REGS_RBP];
855 unsigned si = c->regs[VCPU_REGS_RSI];
856 unsigned di = c->regs[VCPU_REGS_RDI];
858 /* 16-bit ModR/M decode. */
859 switch (c->modrm_mod) {
861 if (c->modrm_rm == 6)
862 c->modrm_ea += insn_fetch(u16, 2, c->eip);
865 c->modrm_ea += insn_fetch(s8, 1, c->eip);
868 c->modrm_ea += insn_fetch(u16, 2, c->eip);
871 switch (c->modrm_rm) {
873 c->modrm_ea += bx + si;
876 c->modrm_ea += bx + di;
879 c->modrm_ea += bp + si;
882 c->modrm_ea += bp + di;
891 if (c->modrm_mod != 0)
898 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
899 (c->modrm_rm == 6 && c->modrm_mod != 0))
900 if (!c->has_seg_override)
901 set_seg_override(c, VCPU_SREG_SS);
902 c->modrm_ea = (u16)c->modrm_ea;
904 /* 32/64-bit ModR/M decode. */
905 if ((c->modrm_rm & 7) == 4) {
906 sib = insn_fetch(u8, 1, c->eip);
907 index_reg |= (sib >> 3) & 7;
911 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
912 c->modrm_ea += insn_fetch(s32, 4, c->eip);
914 c->modrm_ea += c->regs[base_reg];
916 c->modrm_ea += c->regs[index_reg] << scale;
917 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
918 if (ctxt->mode == X86EMUL_MODE_PROT64)
921 c->modrm_ea += c->regs[c->modrm_rm];
922 switch (c->modrm_mod) {
924 if (c->modrm_rm == 5)
925 c->modrm_ea += insn_fetch(s32, 4, c->eip);
928 c->modrm_ea += insn_fetch(s8, 1, c->eip);
931 c->modrm_ea += insn_fetch(s32, 4, c->eip);
939 static int decode_abs(struct x86_emulate_ctxt *ctxt,
940 struct x86_emulate_ops *ops)
942 struct decode_cache *c = &ctxt->decode;
943 int rc = X86EMUL_CONTINUE;
945 switch (c->ad_bytes) {
947 c->modrm_ea = insn_fetch(u16, 2, c->eip);
950 c->modrm_ea = insn_fetch(u32, 4, c->eip);
953 c->modrm_ea = insn_fetch(u64, 8, c->eip);
961 x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
963 struct decode_cache *c = &ctxt->decode;
964 int rc = X86EMUL_CONTINUE;
965 int mode = ctxt->mode;
966 int def_op_bytes, def_ad_bytes, group;
969 /* we cannot decode insn before we complete previous rep insn */
970 WARN_ON(ctxt->restart);
973 c->fetch.start = c->fetch.end = c->eip;
974 ctxt->cs_base = seg_base(ctxt, ops, VCPU_SREG_CS);
977 case X86EMUL_MODE_REAL:
978 case X86EMUL_MODE_VM86:
979 case X86EMUL_MODE_PROT16:
980 def_op_bytes = def_ad_bytes = 2;
982 case X86EMUL_MODE_PROT32:
983 def_op_bytes = def_ad_bytes = 4;
986 case X86EMUL_MODE_PROT64:
995 c->op_bytes = def_op_bytes;
996 c->ad_bytes = def_ad_bytes;
998 /* Legacy prefixes. */
1000 switch (c->b = insn_fetch(u8, 1, c->eip)) {
1001 case 0x66: /* operand-size override */
1002 /* switch between 2/4 bytes */
1003 c->op_bytes = def_op_bytes ^ 6;
1005 case 0x67: /* address-size override */
1006 if (mode == X86EMUL_MODE_PROT64)
1007 /* switch between 4/8 bytes */
1008 c->ad_bytes = def_ad_bytes ^ 12;
1010 /* switch between 2/4 bytes */
1011 c->ad_bytes = def_ad_bytes ^ 6;
1013 case 0x26: /* ES override */
1014 case 0x2e: /* CS override */
1015 case 0x36: /* SS override */
1016 case 0x3e: /* DS override */
1017 set_seg_override(c, (c->b >> 3) & 3);
1019 case 0x64: /* FS override */
1020 case 0x65: /* GS override */
1021 set_seg_override(c, c->b & 7);
1023 case 0x40 ... 0x4f: /* REX */
1024 if (mode != X86EMUL_MODE_PROT64)
1026 c->rex_prefix = c->b;
1028 case 0xf0: /* LOCK */
1031 case 0xf2: /* REPNE/REPNZ */
1032 c->rep_prefix = REPNE_PREFIX;
1034 case 0xf3: /* REP/REPE/REPZ */
1035 c->rep_prefix = REPE_PREFIX;
1041 /* Any legacy prefix after a REX prefix nullifies its effect. */
1050 if (c->rex_prefix & 8)
1051 c->op_bytes = 8; /* REX.W */
1053 /* Opcode byte(s). */
1054 c->d = opcode_table[c->b];
1056 /* Two-byte opcode? */
1059 c->b = insn_fetch(u8, 1, c->eip);
1060 c->d = twobyte_table[c->b];
1065 group = c->d & GroupMask;
1066 c->modrm = insn_fetch(u8, 1, c->eip);
1069 group = (group << 3) + ((c->modrm >> 3) & 7);
1070 if ((c->d & GroupDual) && (c->modrm >> 6) == 3)
1071 c->d = group2_table[group];
1073 c->d = group_table[group];
1078 DPRINTF("Cannot emulate %02x\n", c->b);
1082 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
1085 /* ModRM and SIB bytes. */
1087 rc = decode_modrm(ctxt, ops);
1088 else if (c->d & MemAbs)
1089 rc = decode_abs(ctxt, ops);
1090 if (rc != X86EMUL_CONTINUE)
1093 if (!c->has_seg_override)
1094 set_seg_override(c, VCPU_SREG_DS);
1096 if (!(!c->twobyte && c->b == 0x8d))
1097 c->modrm_ea += seg_override_base(ctxt, ops, c);
1099 if (c->ad_bytes != 8)
1100 c->modrm_ea = (u32)c->modrm_ea;
1102 if (c->rip_relative)
1103 c->modrm_ea += c->eip;
1106 * Decode and fetch the source operand: register, memory
1109 switch (c->d & SrcMask) {
1113 decode_register_operand(&c->src, c, 0);
1122 c->src.bytes = (c->d & ByteOp) ? 1 :
1124 /* Don't fetch the address for invlpg: it could be unmapped. */
1125 if (c->twobyte && c->b == 0x01 && c->modrm_reg == 7)
1129 * For instructions with a ModR/M byte, switch to register
1130 * access if Mod = 3.
1132 if ((c->d & ModRM) && c->modrm_mod == 3) {
1133 c->src.type = OP_REG;
1134 c->src.val = c->modrm_val;
1135 c->src.ptr = c->modrm_ptr;
1138 c->src.type = OP_MEM;
1139 c->src.ptr = (unsigned long *)c->modrm_ea;
1144 c->src.type = OP_IMM;
1145 c->src.ptr = (unsigned long *)c->eip;
1146 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1147 if (c->src.bytes == 8)
1149 /* NB. Immediates are sign-extended as necessary. */
1150 switch (c->src.bytes) {
1152 c->src.val = insn_fetch(s8, 1, c->eip);
1155 c->src.val = insn_fetch(s16, 2, c->eip);
1158 c->src.val = insn_fetch(s32, 4, c->eip);
1161 if ((c->d & SrcMask) == SrcImmU) {
1162 switch (c->src.bytes) {
1167 c->src.val &= 0xffff;
1170 c->src.val &= 0xffffffff;
1177 c->src.type = OP_IMM;
1178 c->src.ptr = (unsigned long *)c->eip;
1180 if ((c->d & SrcMask) == SrcImmByte)
1181 c->src.val = insn_fetch(s8, 1, c->eip);
1183 c->src.val = insn_fetch(u8, 1, c->eip);
1186 c->src.type = OP_REG;
1187 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1188 c->src.ptr = &c->regs[VCPU_REGS_RAX];
1189 switch (c->src.bytes) {
1191 c->src.val = *(u8 *)c->src.ptr;
1194 c->src.val = *(u16 *)c->src.ptr;
1197 c->src.val = *(u32 *)c->src.ptr;
1200 c->src.val = *(u64 *)c->src.ptr;
1209 c->src.type = OP_MEM;
1210 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1211 c->src.ptr = (unsigned long *)
1212 register_address(c, seg_override_base(ctxt, ops, c),
1213 c->regs[VCPU_REGS_RSI]);
1217 c->src.type = OP_IMM;
1218 c->src.ptr = (unsigned long *)c->eip;
1219 c->src.bytes = c->op_bytes + 2;
1220 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
1223 c->src.type = OP_MEM;
1224 c->src.ptr = (unsigned long *)c->modrm_ea;
1225 c->src.bytes = c->op_bytes + 2;
1230 * Decode and fetch the second source operand: register, memory
1233 switch (c->d & Src2Mask) {
1238 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
1241 c->src2.type = OP_IMM;
1242 c->src2.ptr = (unsigned long *)c->eip;
1244 c->src2.val = insn_fetch(u8, 1, c->eip);
1252 /* Decode and fetch the destination operand: register or memory. */
1253 switch (c->d & DstMask) {
1255 /* Special instructions do their own operand decoding. */
1258 decode_register_operand(&c->dst, c,
1259 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
1263 if ((c->d & ModRM) && c->modrm_mod == 3) {
1264 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1265 c->dst.type = OP_REG;
1266 c->dst.val = c->dst.orig_val = c->modrm_val;
1267 c->dst.ptr = c->modrm_ptr;
1270 c->dst.type = OP_MEM;
1271 c->dst.ptr = (unsigned long *)c->modrm_ea;
1272 if ((c->d & DstMask) == DstMem64)
1275 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1278 unsigned long mask = ~(c->dst.bytes * 8 - 1);
1280 c->dst.ptr = (void *)c->dst.ptr +
1281 (c->src.val & mask) / 8;
1285 c->dst.type = OP_REG;
1286 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1287 c->dst.ptr = &c->regs[VCPU_REGS_RAX];
1288 switch (c->dst.bytes) {
1290 c->dst.val = *(u8 *)c->dst.ptr;
1293 c->dst.val = *(u16 *)c->dst.ptr;
1296 c->dst.val = *(u32 *)c->dst.ptr;
1299 c->dst.val = *(u64 *)c->dst.ptr;
1302 c->dst.orig_val = c->dst.val;
1305 c->dst.type = OP_MEM;
1306 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1307 c->dst.ptr = (unsigned long *)
1308 register_address(c, es_base(ctxt, ops),
1309 c->regs[VCPU_REGS_RDI]);
1315 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
1318 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1319 struct x86_emulate_ops *ops,
1320 unsigned long addr, void *dest, unsigned size)
1323 struct read_cache *mc = &ctxt->decode.mem_read;
1327 int n = min(size, 8u);
1329 if (mc->pos < mc->end)
1332 rc = ops->read_emulated(addr, mc->data + mc->end, n, &err,
1334 if (rc == X86EMUL_PROPAGATE_FAULT)
1335 emulate_pf(ctxt, addr, err);
1336 if (rc != X86EMUL_CONTINUE)
1341 memcpy(dest, mc->data + mc->pos, n);
1346 return X86EMUL_CONTINUE;
1349 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1350 struct x86_emulate_ops *ops,
1351 unsigned int size, unsigned short port,
1354 struct read_cache *rc = &ctxt->decode.io_read;
1356 if (rc->pos == rc->end) { /* refill pio read ahead */
1357 struct decode_cache *c = &ctxt->decode;
1358 unsigned int in_page, n;
1359 unsigned int count = c->rep_prefix ?
1360 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
1361 in_page = (ctxt->eflags & EFLG_DF) ?
1362 offset_in_page(c->regs[VCPU_REGS_RDI]) :
1363 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
1364 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1368 rc->pos = rc->end = 0;
1369 if (!ops->pio_in_emulated(size, port, rc->data, n, ctxt->vcpu))
1374 memcpy(dest, rc->data + rc->pos, size);
1379 static u32 desc_limit_scaled(struct desc_struct *desc)
1381 u32 limit = get_desc_limit(desc);
1383 return desc->g ? (limit << 12) | 0xfff : limit;
1386 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1387 struct x86_emulate_ops *ops,
1388 u16 selector, struct desc_ptr *dt)
1390 if (selector & 1 << 2) {
1391 struct desc_struct desc;
1392 memset (dt, 0, sizeof *dt);
1393 if (!ops->get_cached_descriptor(&desc, VCPU_SREG_LDTR, ctxt->vcpu))
1396 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1397 dt->address = get_desc_base(&desc);
1399 ops->get_gdt(dt, ctxt->vcpu);
1402 /* allowed just for 8 bytes segments */
1403 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1404 struct x86_emulate_ops *ops,
1405 u16 selector, struct desc_struct *desc)
1408 u16 index = selector >> 3;
1413 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1415 if (dt.size < index * 8 + 7) {
1416 emulate_gp(ctxt, selector & 0xfffc);
1417 return X86EMUL_PROPAGATE_FAULT;
1419 addr = dt.address + index * 8;
1420 ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu, &err);
1421 if (ret == X86EMUL_PROPAGATE_FAULT)
1422 emulate_pf(ctxt, addr, err);
1427 /* allowed just for 8 bytes segments */
1428 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1429 struct x86_emulate_ops *ops,
1430 u16 selector, struct desc_struct *desc)
1433 u16 index = selector >> 3;
1438 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1440 if (dt.size < index * 8 + 7) {
1441 emulate_gp(ctxt, selector & 0xfffc);
1442 return X86EMUL_PROPAGATE_FAULT;
1445 addr = dt.address + index * 8;
1446 ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu, &err);
1447 if (ret == X86EMUL_PROPAGATE_FAULT)
1448 emulate_pf(ctxt, addr, err);
1453 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1454 struct x86_emulate_ops *ops,
1455 u16 selector, int seg)
1457 struct desc_struct seg_desc;
1459 unsigned err_vec = GP_VECTOR;
1461 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1464 memset(&seg_desc, 0, sizeof seg_desc);
1466 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1467 || ctxt->mode == X86EMUL_MODE_REAL) {
1468 /* set real mode segment descriptor */
1469 set_desc_base(&seg_desc, selector << 4);
1470 set_desc_limit(&seg_desc, 0xffff);
1477 /* NULL selector is not valid for TR, CS and SS */
1478 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1482 /* TR should be in GDT only */
1483 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1486 if (null_selector) /* for NULL selector skip all following checks */
1489 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1490 if (ret != X86EMUL_CONTINUE)
1493 err_code = selector & 0xfffc;
1494 err_vec = GP_VECTOR;
1496 /* can't load system descriptor into segment selecor */
1497 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1501 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1507 cpl = ops->cpl(ctxt->vcpu);
1512 * segment is not a writable data segment or segment
1513 * selector's RPL != CPL or segment selector's RPL != CPL
1515 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1519 if (!(seg_desc.type & 8))
1522 if (seg_desc.type & 4) {
1528 if (rpl > cpl || dpl != cpl)
1531 /* CS(RPL) <- CPL */
1532 selector = (selector & 0xfffc) | cpl;
1535 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1538 case VCPU_SREG_LDTR:
1539 if (seg_desc.s || seg_desc.type != 2)
1542 default: /* DS, ES, FS, or GS */
1544 * segment is not a data or readable code segment or
1545 * ((segment is a data or nonconforming code segment)
1546 * and (both RPL and CPL > DPL))
1548 if ((seg_desc.type & 0xa) == 0x8 ||
1549 (((seg_desc.type & 0xc) != 0xc) &&
1550 (rpl > dpl && cpl > dpl)))
1556 /* mark segment as accessed */
1558 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1559 if (ret != X86EMUL_CONTINUE)
1563 ops->set_segment_selector(selector, seg, ctxt->vcpu);
1564 ops->set_cached_descriptor(&seg_desc, seg, ctxt->vcpu);
1565 return X86EMUL_CONTINUE;
1567 emulate_exception(ctxt, err_vec, err_code, true);
1568 return X86EMUL_PROPAGATE_FAULT;
1571 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1572 struct x86_emulate_ops *ops)
1575 struct decode_cache *c = &ctxt->decode;
1578 switch (c->dst.type) {
1580 /* The 4-byte case *is* correct:
1581 * in 64-bit mode we zero-extend.
1583 switch (c->dst.bytes) {
1585 *(u8 *)c->dst.ptr = (u8)c->dst.val;
1588 *(u16 *)c->dst.ptr = (u16)c->dst.val;
1591 *c->dst.ptr = (u32)c->dst.val;
1592 break; /* 64b: zero-ext */
1594 *c->dst.ptr = c->dst.val;
1600 rc = ops->cmpxchg_emulated(
1601 (unsigned long)c->dst.ptr,
1608 rc = ops->write_emulated(
1609 (unsigned long)c->dst.ptr,
1614 if (rc == X86EMUL_PROPAGATE_FAULT)
1616 (unsigned long)c->dst.ptr, err);
1617 if (rc != X86EMUL_CONTINUE)
1626 return X86EMUL_CONTINUE;
1629 static inline void emulate_push(struct x86_emulate_ctxt *ctxt,
1630 struct x86_emulate_ops *ops)
1632 struct decode_cache *c = &ctxt->decode;
1634 c->dst.type = OP_MEM;
1635 c->dst.bytes = c->op_bytes;
1636 c->dst.val = c->src.val;
1637 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1638 c->dst.ptr = (void *) register_address(c, ss_base(ctxt, ops),
1639 c->regs[VCPU_REGS_RSP]);
1642 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1643 struct x86_emulate_ops *ops,
1644 void *dest, int len)
1646 struct decode_cache *c = &ctxt->decode;
1649 rc = read_emulated(ctxt, ops, register_address(c, ss_base(ctxt, ops),
1650 c->regs[VCPU_REGS_RSP]),
1652 if (rc != X86EMUL_CONTINUE)
1655 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1659 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1660 struct x86_emulate_ops *ops,
1661 void *dest, int len)
1664 unsigned long val, change_mask;
1665 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1666 int cpl = ops->cpl(ctxt->vcpu);
1668 rc = emulate_pop(ctxt, ops, &val, len);
1669 if (rc != X86EMUL_CONTINUE)
1672 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1673 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1675 switch(ctxt->mode) {
1676 case X86EMUL_MODE_PROT64:
1677 case X86EMUL_MODE_PROT32:
1678 case X86EMUL_MODE_PROT16:
1680 change_mask |= EFLG_IOPL;
1682 change_mask |= EFLG_IF;
1684 case X86EMUL_MODE_VM86:
1686 emulate_gp(ctxt, 0);
1687 return X86EMUL_PROPAGATE_FAULT;
1689 change_mask |= EFLG_IF;
1691 default: /* real mode */
1692 change_mask |= (EFLG_IOPL | EFLG_IF);
1696 *(unsigned long *)dest =
1697 (ctxt->eflags & ~change_mask) | (val & change_mask);
1702 static void emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1703 struct x86_emulate_ops *ops, int seg)
1705 struct decode_cache *c = &ctxt->decode;
1707 c->src.val = ops->get_segment_selector(seg, ctxt->vcpu);
1709 emulate_push(ctxt, ops);
1712 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1713 struct x86_emulate_ops *ops, int seg)
1715 struct decode_cache *c = &ctxt->decode;
1716 unsigned long selector;
1719 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1720 if (rc != X86EMUL_CONTINUE)
1723 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1727 static int emulate_pusha(struct x86_emulate_ctxt *ctxt,
1728 struct x86_emulate_ops *ops)
1730 struct decode_cache *c = &ctxt->decode;
1731 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1732 int rc = X86EMUL_CONTINUE;
1733 int reg = VCPU_REGS_RAX;
1735 while (reg <= VCPU_REGS_RDI) {
1736 (reg == VCPU_REGS_RSP) ?
1737 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1739 emulate_push(ctxt, ops);
1741 rc = writeback(ctxt, ops);
1742 if (rc != X86EMUL_CONTINUE)
1748 /* Disable writeback. */
1749 c->dst.type = OP_NONE;
1754 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1755 struct x86_emulate_ops *ops)
1757 struct decode_cache *c = &ctxt->decode;
1758 int rc = X86EMUL_CONTINUE;
1759 int reg = VCPU_REGS_RDI;
1761 while (reg >= VCPU_REGS_RAX) {
1762 if (reg == VCPU_REGS_RSP) {
1763 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1768 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1769 if (rc != X86EMUL_CONTINUE)
1776 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1777 struct x86_emulate_ops *ops)
1779 struct decode_cache *c = &ctxt->decode;
1781 return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1784 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1786 struct decode_cache *c = &ctxt->decode;
1787 switch (c->modrm_reg) {
1789 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1792 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1795 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1798 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1800 case 4: /* sal/shl */
1801 case 6: /* sal/shl */
1802 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1805 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1808 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1813 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1814 struct x86_emulate_ops *ops)
1816 struct decode_cache *c = &ctxt->decode;
1818 switch (c->modrm_reg) {
1819 case 0 ... 1: /* test */
1820 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1823 c->dst.val = ~c->dst.val;
1826 emulate_1op("neg", c->dst, ctxt->eflags);
1834 static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
1835 struct x86_emulate_ops *ops)
1837 struct decode_cache *c = &ctxt->decode;
1839 switch (c->modrm_reg) {
1841 emulate_1op("inc", c->dst, ctxt->eflags);
1844 emulate_1op("dec", c->dst, ctxt->eflags);
1846 case 2: /* call near abs */ {
1849 c->eip = c->src.val;
1850 c->src.val = old_eip;
1851 emulate_push(ctxt, ops);
1854 case 4: /* jmp abs */
1855 c->eip = c->src.val;
1858 emulate_push(ctxt, ops);
1861 return X86EMUL_CONTINUE;
1864 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1865 struct x86_emulate_ops *ops)
1867 struct decode_cache *c = &ctxt->decode;
1868 u64 old = c->dst.orig_val64;
1870 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1871 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1872 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1873 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1874 ctxt->eflags &= ~EFLG_ZF;
1876 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1877 (u32) c->regs[VCPU_REGS_RBX];
1879 ctxt->eflags |= EFLG_ZF;
1881 return X86EMUL_CONTINUE;
1884 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1885 struct x86_emulate_ops *ops)
1887 struct decode_cache *c = &ctxt->decode;
1891 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1892 if (rc != X86EMUL_CONTINUE)
1894 if (c->op_bytes == 4)
1895 c->eip = (u32)c->eip;
1896 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1897 if (rc != X86EMUL_CONTINUE)
1899 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1904 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1905 struct x86_emulate_ops *ops, struct desc_struct *cs,
1906 struct desc_struct *ss)
1908 memset(cs, 0, sizeof(struct desc_struct));
1909 ops->get_cached_descriptor(cs, VCPU_SREG_CS, ctxt->vcpu);
1910 memset(ss, 0, sizeof(struct desc_struct));
1912 cs->l = 0; /* will be adjusted later */
1913 set_desc_base(cs, 0); /* flat segment */
1914 cs->g = 1; /* 4kb granularity */
1915 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1916 cs->type = 0x0b; /* Read, Execute, Accessed */
1918 cs->dpl = 0; /* will be adjusted later */
1922 set_desc_base(ss, 0); /* flat segment */
1923 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1924 ss->g = 1; /* 4kb granularity */
1926 ss->type = 0x03; /* Read/Write, Accessed */
1927 ss->d = 1; /* 32bit stack segment */
1933 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1935 struct decode_cache *c = &ctxt->decode;
1936 struct desc_struct cs, ss;
1940 /* syscall is not available in real mode */
1941 if (ctxt->mode == X86EMUL_MODE_REAL ||
1942 ctxt->mode == X86EMUL_MODE_VM86) {
1944 return X86EMUL_PROPAGATE_FAULT;
1947 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1949 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1951 cs_sel = (u16)(msr_data & 0xfffc);
1952 ss_sel = (u16)(msr_data + 8);
1954 if (is_long_mode(ctxt->vcpu)) {
1958 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
1959 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1960 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
1961 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1963 c->regs[VCPU_REGS_RCX] = c->eip;
1964 if (is_long_mode(ctxt->vcpu)) {
1965 #ifdef CONFIG_X86_64
1966 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1968 ops->get_msr(ctxt->vcpu,
1969 ctxt->mode == X86EMUL_MODE_PROT64 ?
1970 MSR_LSTAR : MSR_CSTAR, &msr_data);
1973 ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
1974 ctxt->eflags &= ~(msr_data | EFLG_RF);
1978 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1979 c->eip = (u32)msr_data;
1981 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1984 return X86EMUL_CONTINUE;
1988 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1990 struct decode_cache *c = &ctxt->decode;
1991 struct desc_struct cs, ss;
1995 /* inject #GP if in real mode */
1996 if (ctxt->mode == X86EMUL_MODE_REAL) {
1997 emulate_gp(ctxt, 0);
1998 return X86EMUL_PROPAGATE_FAULT;
2001 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2002 * Therefore, we inject an #UD.
2004 if (ctxt->mode == X86EMUL_MODE_PROT64) {
2006 return X86EMUL_PROPAGATE_FAULT;
2009 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2011 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
2012 switch (ctxt->mode) {
2013 case X86EMUL_MODE_PROT32:
2014 if ((msr_data & 0xfffc) == 0x0) {
2015 emulate_gp(ctxt, 0);
2016 return X86EMUL_PROPAGATE_FAULT;
2019 case X86EMUL_MODE_PROT64:
2020 if (msr_data == 0x0) {
2021 emulate_gp(ctxt, 0);
2022 return X86EMUL_PROPAGATE_FAULT;
2027 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2028 cs_sel = (u16)msr_data;
2029 cs_sel &= ~SELECTOR_RPL_MASK;
2030 ss_sel = cs_sel + 8;
2031 ss_sel &= ~SELECTOR_RPL_MASK;
2032 if (ctxt->mode == X86EMUL_MODE_PROT64
2033 || is_long_mode(ctxt->vcpu)) {
2038 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
2039 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
2040 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
2041 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
2043 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
2046 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
2047 c->regs[VCPU_REGS_RSP] = msr_data;
2049 return X86EMUL_CONTINUE;
2053 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2055 struct decode_cache *c = &ctxt->decode;
2056 struct desc_struct cs, ss;
2061 /* inject #GP if in real mode or Virtual 8086 mode */
2062 if (ctxt->mode == X86EMUL_MODE_REAL ||
2063 ctxt->mode == X86EMUL_MODE_VM86) {
2064 emulate_gp(ctxt, 0);
2065 return X86EMUL_PROPAGATE_FAULT;
2068 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2070 if ((c->rex_prefix & 0x8) != 0x0)
2071 usermode = X86EMUL_MODE_PROT64;
2073 usermode = X86EMUL_MODE_PROT32;
2077 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
2079 case X86EMUL_MODE_PROT32:
2080 cs_sel = (u16)(msr_data + 16);
2081 if ((msr_data & 0xfffc) == 0x0) {
2082 emulate_gp(ctxt, 0);
2083 return X86EMUL_PROPAGATE_FAULT;
2085 ss_sel = (u16)(msr_data + 24);
2087 case X86EMUL_MODE_PROT64:
2088 cs_sel = (u16)(msr_data + 32);
2089 if (msr_data == 0x0) {
2090 emulate_gp(ctxt, 0);
2091 return X86EMUL_PROPAGATE_FAULT;
2093 ss_sel = cs_sel + 8;
2098 cs_sel |= SELECTOR_RPL_MASK;
2099 ss_sel |= SELECTOR_RPL_MASK;
2101 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
2102 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
2103 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
2104 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
2106 c->eip = c->regs[VCPU_REGS_RDX];
2107 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
2109 return X86EMUL_CONTINUE;
2112 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
2113 struct x86_emulate_ops *ops)
2116 if (ctxt->mode == X86EMUL_MODE_REAL)
2118 if (ctxt->mode == X86EMUL_MODE_VM86)
2120 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2121 return ops->cpl(ctxt->vcpu) > iopl;
2124 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2125 struct x86_emulate_ops *ops,
2128 struct desc_struct tr_seg;
2131 u8 perm, bit_idx = port & 0x7;
2132 unsigned mask = (1 << len) - 1;
2134 ops->get_cached_descriptor(&tr_seg, VCPU_SREG_TR, ctxt->vcpu);
2137 if (desc_limit_scaled(&tr_seg) < 103)
2139 r = ops->read_std(get_desc_base(&tr_seg) + 102, &io_bitmap_ptr, 2,
2141 if (r != X86EMUL_CONTINUE)
2143 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2145 r = ops->read_std(get_desc_base(&tr_seg) + io_bitmap_ptr + port/8,
2146 &perm, 1, ctxt->vcpu, NULL);
2147 if (r != X86EMUL_CONTINUE)
2149 if ((perm >> bit_idx) & mask)
2154 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2155 struct x86_emulate_ops *ops,
2158 if (emulator_bad_iopl(ctxt, ops))
2159 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
2164 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2165 struct x86_emulate_ops *ops,
2166 struct tss_segment_16 *tss)
2168 struct decode_cache *c = &ctxt->decode;
2171 tss->flag = ctxt->eflags;
2172 tss->ax = c->regs[VCPU_REGS_RAX];
2173 tss->cx = c->regs[VCPU_REGS_RCX];
2174 tss->dx = c->regs[VCPU_REGS_RDX];
2175 tss->bx = c->regs[VCPU_REGS_RBX];
2176 tss->sp = c->regs[VCPU_REGS_RSP];
2177 tss->bp = c->regs[VCPU_REGS_RBP];
2178 tss->si = c->regs[VCPU_REGS_RSI];
2179 tss->di = c->regs[VCPU_REGS_RDI];
2181 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2182 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2183 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2184 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2185 tss->ldt = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2188 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2189 struct x86_emulate_ops *ops,
2190 struct tss_segment_16 *tss)
2192 struct decode_cache *c = &ctxt->decode;
2196 ctxt->eflags = tss->flag | 2;
2197 c->regs[VCPU_REGS_RAX] = tss->ax;
2198 c->regs[VCPU_REGS_RCX] = tss->cx;
2199 c->regs[VCPU_REGS_RDX] = tss->dx;
2200 c->regs[VCPU_REGS_RBX] = tss->bx;
2201 c->regs[VCPU_REGS_RSP] = tss->sp;
2202 c->regs[VCPU_REGS_RBP] = tss->bp;
2203 c->regs[VCPU_REGS_RSI] = tss->si;
2204 c->regs[VCPU_REGS_RDI] = tss->di;
2207 * SDM says that segment selectors are loaded before segment
2210 ops->set_segment_selector(tss->ldt, VCPU_SREG_LDTR, ctxt->vcpu);
2211 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2212 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2213 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2214 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2217 * Now load segment descriptors. If fault happenes at this stage
2218 * it is handled in a context of new task
2220 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
2221 if (ret != X86EMUL_CONTINUE)
2223 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2224 if (ret != X86EMUL_CONTINUE)
2226 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2227 if (ret != X86EMUL_CONTINUE)
2229 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2230 if (ret != X86EMUL_CONTINUE)
2232 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2233 if (ret != X86EMUL_CONTINUE)
2236 return X86EMUL_CONTINUE;
2239 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2240 struct x86_emulate_ops *ops,
2241 u16 tss_selector, u16 old_tss_sel,
2242 ulong old_tss_base, struct desc_struct *new_desc)
2244 struct tss_segment_16 tss_seg;
2246 u32 err, new_tss_base = get_desc_base(new_desc);
2248 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2250 if (ret == X86EMUL_PROPAGATE_FAULT) {
2251 /* FIXME: need to provide precise fault address */
2252 emulate_pf(ctxt, old_tss_base, err);
2256 save_state_to_tss16(ctxt, ops, &tss_seg);
2258 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2260 if (ret == X86EMUL_PROPAGATE_FAULT) {
2261 /* FIXME: need to provide precise fault address */
2262 emulate_pf(ctxt, old_tss_base, err);
2266 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2268 if (ret == X86EMUL_PROPAGATE_FAULT) {
2269 /* FIXME: need to provide precise fault address */
2270 emulate_pf(ctxt, new_tss_base, err);
2274 if (old_tss_sel != 0xffff) {
2275 tss_seg.prev_task_link = old_tss_sel;
2277 ret = ops->write_std(new_tss_base,
2278 &tss_seg.prev_task_link,
2279 sizeof tss_seg.prev_task_link,
2281 if (ret == X86EMUL_PROPAGATE_FAULT) {
2282 /* FIXME: need to provide precise fault address */
2283 emulate_pf(ctxt, new_tss_base, err);
2288 return load_state_from_tss16(ctxt, ops, &tss_seg);
2291 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2292 struct x86_emulate_ops *ops,
2293 struct tss_segment_32 *tss)
2295 struct decode_cache *c = &ctxt->decode;
2297 tss->cr3 = ops->get_cr(3, ctxt->vcpu);
2299 tss->eflags = ctxt->eflags;
2300 tss->eax = c->regs[VCPU_REGS_RAX];
2301 tss->ecx = c->regs[VCPU_REGS_RCX];
2302 tss->edx = c->regs[VCPU_REGS_RDX];
2303 tss->ebx = c->regs[VCPU_REGS_RBX];
2304 tss->esp = c->regs[VCPU_REGS_RSP];
2305 tss->ebp = c->regs[VCPU_REGS_RBP];
2306 tss->esi = c->regs[VCPU_REGS_RSI];
2307 tss->edi = c->regs[VCPU_REGS_RDI];
2309 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2310 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2311 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2312 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2313 tss->fs = ops->get_segment_selector(VCPU_SREG_FS, ctxt->vcpu);
2314 tss->gs = ops->get_segment_selector(VCPU_SREG_GS, ctxt->vcpu);
2315 tss->ldt_selector = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2318 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2319 struct x86_emulate_ops *ops,
2320 struct tss_segment_32 *tss)
2322 struct decode_cache *c = &ctxt->decode;
2325 if (ops->set_cr(3, tss->cr3, ctxt->vcpu)) {
2326 emulate_gp(ctxt, 0);
2327 return X86EMUL_PROPAGATE_FAULT;
2330 ctxt->eflags = tss->eflags | 2;
2331 c->regs[VCPU_REGS_RAX] = tss->eax;
2332 c->regs[VCPU_REGS_RCX] = tss->ecx;
2333 c->regs[VCPU_REGS_RDX] = tss->edx;
2334 c->regs[VCPU_REGS_RBX] = tss->ebx;
2335 c->regs[VCPU_REGS_RSP] = tss->esp;
2336 c->regs[VCPU_REGS_RBP] = tss->ebp;
2337 c->regs[VCPU_REGS_RSI] = tss->esi;
2338 c->regs[VCPU_REGS_RDI] = tss->edi;
2341 * SDM says that segment selectors are loaded before segment
2344 ops->set_segment_selector(tss->ldt_selector, VCPU_SREG_LDTR, ctxt->vcpu);
2345 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2346 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2347 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2348 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2349 ops->set_segment_selector(tss->fs, VCPU_SREG_FS, ctxt->vcpu);
2350 ops->set_segment_selector(tss->gs, VCPU_SREG_GS, ctxt->vcpu);
2353 * Now load segment descriptors. If fault happenes at this stage
2354 * it is handled in a context of new task
2356 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2357 if (ret != X86EMUL_CONTINUE)
2359 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2360 if (ret != X86EMUL_CONTINUE)
2362 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2363 if (ret != X86EMUL_CONTINUE)
2365 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2366 if (ret != X86EMUL_CONTINUE)
2368 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2369 if (ret != X86EMUL_CONTINUE)
2371 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2372 if (ret != X86EMUL_CONTINUE)
2374 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2375 if (ret != X86EMUL_CONTINUE)
2378 return X86EMUL_CONTINUE;
2381 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2382 struct x86_emulate_ops *ops,
2383 u16 tss_selector, u16 old_tss_sel,
2384 ulong old_tss_base, struct desc_struct *new_desc)
2386 struct tss_segment_32 tss_seg;
2388 u32 err, new_tss_base = get_desc_base(new_desc);
2390 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2392 if (ret == X86EMUL_PROPAGATE_FAULT) {
2393 /* FIXME: need to provide precise fault address */
2394 emulate_pf(ctxt, old_tss_base, err);
2398 save_state_to_tss32(ctxt, ops, &tss_seg);
2400 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2402 if (ret == X86EMUL_PROPAGATE_FAULT) {
2403 /* FIXME: need to provide precise fault address */
2404 emulate_pf(ctxt, old_tss_base, err);
2408 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2410 if (ret == X86EMUL_PROPAGATE_FAULT) {
2411 /* FIXME: need to provide precise fault address */
2412 emulate_pf(ctxt, new_tss_base, err);
2416 if (old_tss_sel != 0xffff) {
2417 tss_seg.prev_task_link = old_tss_sel;
2419 ret = ops->write_std(new_tss_base,
2420 &tss_seg.prev_task_link,
2421 sizeof tss_seg.prev_task_link,
2423 if (ret == X86EMUL_PROPAGATE_FAULT) {
2424 /* FIXME: need to provide precise fault address */
2425 emulate_pf(ctxt, new_tss_base, err);
2430 return load_state_from_tss32(ctxt, ops, &tss_seg);
2433 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2434 struct x86_emulate_ops *ops,
2435 u16 tss_selector, int reason,
2436 bool has_error_code, u32 error_code)
2438 struct desc_struct curr_tss_desc, next_tss_desc;
2440 u16 old_tss_sel = ops->get_segment_selector(VCPU_SREG_TR, ctxt->vcpu);
2441 ulong old_tss_base =
2442 ops->get_cached_segment_base(VCPU_SREG_TR, ctxt->vcpu);
2445 /* FIXME: old_tss_base == ~0 ? */
2447 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2448 if (ret != X86EMUL_CONTINUE)
2450 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2451 if (ret != X86EMUL_CONTINUE)
2454 /* FIXME: check that next_tss_desc is tss */
2456 if (reason != TASK_SWITCH_IRET) {
2457 if ((tss_selector & 3) > next_tss_desc.dpl ||
2458 ops->cpl(ctxt->vcpu) > next_tss_desc.dpl) {
2459 emulate_gp(ctxt, 0);
2460 return X86EMUL_PROPAGATE_FAULT;
2464 desc_limit = desc_limit_scaled(&next_tss_desc);
2465 if (!next_tss_desc.p ||
2466 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2467 desc_limit < 0x2b)) {
2468 emulate_ts(ctxt, tss_selector & 0xfffc);
2469 return X86EMUL_PROPAGATE_FAULT;
2472 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2473 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2474 write_segment_descriptor(ctxt, ops, old_tss_sel,
2478 if (reason == TASK_SWITCH_IRET)
2479 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2481 /* set back link to prev task only if NT bit is set in eflags
2482 note that old_tss_sel is not used afetr this point */
2483 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2484 old_tss_sel = 0xffff;
2486 if (next_tss_desc.type & 8)
2487 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2488 old_tss_base, &next_tss_desc);
2490 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2491 old_tss_base, &next_tss_desc);
2492 if (ret != X86EMUL_CONTINUE)
2495 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2496 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2498 if (reason != TASK_SWITCH_IRET) {
2499 next_tss_desc.type |= (1 << 1); /* set busy flag */
2500 write_segment_descriptor(ctxt, ops, tss_selector,
2504 ops->set_cr(0, ops->get_cr(0, ctxt->vcpu) | X86_CR0_TS, ctxt->vcpu);
2505 ops->set_cached_descriptor(&next_tss_desc, VCPU_SREG_TR, ctxt->vcpu);
2506 ops->set_segment_selector(tss_selector, VCPU_SREG_TR, ctxt->vcpu);
2508 if (has_error_code) {
2509 struct decode_cache *c = &ctxt->decode;
2511 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2513 c->src.val = (unsigned long) error_code;
2514 emulate_push(ctxt, ops);
2520 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2521 struct x86_emulate_ops *ops,
2522 u16 tss_selector, int reason,
2523 bool has_error_code, u32 error_code)
2525 struct decode_cache *c = &ctxt->decode;
2529 c->dst.type = OP_NONE;
2531 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2532 has_error_code, error_code);
2534 if (rc == X86EMUL_CONTINUE) {
2535 rc = writeback(ctxt, ops);
2536 if (rc == X86EMUL_CONTINUE)
2540 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
2543 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned long base,
2544 int reg, struct operand *op)
2546 struct decode_cache *c = &ctxt->decode;
2547 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2549 register_address_increment(c, &c->regs[reg], df * op->bytes);
2550 op->ptr = (unsigned long *)register_address(c, base, c->regs[reg]);
2554 x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2557 struct decode_cache *c = &ctxt->decode;
2558 int rc = X86EMUL_CONTINUE;
2559 int saved_dst_type = c->dst.type;
2561 ctxt->decode.mem_read.pos = 0;
2563 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
2568 /* LOCK prefix is allowed only with some instructions */
2569 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
2574 /* Privileged instruction can be executed only in CPL=0 */
2575 if ((c->d & Priv) && ops->cpl(ctxt->vcpu)) {
2576 emulate_gp(ctxt, 0);
2580 if (c->rep_prefix && (c->d & String)) {
2581 ctxt->restart = true;
2582 /* All REP prefixes have the same first termination condition */
2583 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
2585 ctxt->restart = false;
2589 /* The second termination condition only applies for REPE
2590 * and REPNE. Test if the repeat string operation prefix is
2591 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
2592 * corresponding termination condition according to:
2593 * - if REPE/REPZ and ZF = 0 then done
2594 * - if REPNE/REPNZ and ZF = 1 then done
2596 if ((c->b == 0xa6) || (c->b == 0xa7) ||
2597 (c->b == 0xae) || (c->b == 0xaf)) {
2598 if ((c->rep_prefix == REPE_PREFIX) &&
2599 ((ctxt->eflags & EFLG_ZF) == 0))
2601 if ((c->rep_prefix == REPNE_PREFIX) &&
2602 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))
2608 if (c->src.type == OP_MEM) {
2609 rc = read_emulated(ctxt, ops, (unsigned long)c->src.ptr,
2610 c->src.valptr, c->src.bytes);
2611 if (rc != X86EMUL_CONTINUE)
2613 c->src.orig_val64 = c->src.val64;
2616 if (c->src2.type == OP_MEM) {
2617 rc = read_emulated(ctxt, ops, (unsigned long)c->src2.ptr,
2618 &c->src2.val, c->src2.bytes);
2619 if (rc != X86EMUL_CONTINUE)
2623 if ((c->d & DstMask) == ImplicitOps)
2627 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
2628 /* optimisation - avoid slow emulated read if Mov */
2629 rc = read_emulated(ctxt, ops, (unsigned long)c->dst.ptr,
2630 &c->dst.val, c->dst.bytes);
2631 if (rc != X86EMUL_CONTINUE)
2634 c->dst.orig_val = c->dst.val;
2644 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
2646 case 0x06: /* push es */
2647 emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
2649 case 0x07: /* pop es */
2650 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
2651 if (rc != X86EMUL_CONTINUE)
2656 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2658 case 0x0e: /* push cs */
2659 emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
2663 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
2665 case 0x16: /* push ss */
2666 emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
2668 case 0x17: /* pop ss */
2669 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
2670 if (rc != X86EMUL_CONTINUE)
2675 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
2677 case 0x1e: /* push ds */
2678 emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
2680 case 0x1f: /* pop ds */
2681 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
2682 if (rc != X86EMUL_CONTINUE)
2687 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
2691 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
2695 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
2699 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2701 case 0x40 ... 0x47: /* inc r16/r32 */
2702 emulate_1op("inc", c->dst, ctxt->eflags);
2704 case 0x48 ... 0x4f: /* dec r16/r32 */
2705 emulate_1op("dec", c->dst, ctxt->eflags);
2707 case 0x50 ... 0x57: /* push reg */
2708 emulate_push(ctxt, ops);
2710 case 0x58 ... 0x5f: /* pop reg */
2712 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
2713 if (rc != X86EMUL_CONTINUE)
2716 case 0x60: /* pusha */
2717 rc = emulate_pusha(ctxt, ops);
2718 if (rc != X86EMUL_CONTINUE)
2721 case 0x61: /* popa */
2722 rc = emulate_popa(ctxt, ops);
2723 if (rc != X86EMUL_CONTINUE)
2726 case 0x63: /* movsxd */
2727 if (ctxt->mode != X86EMUL_MODE_PROT64)
2728 goto cannot_emulate;
2729 c->dst.val = (s32) c->src.val;
2731 case 0x68: /* push imm */
2732 case 0x6a: /* push imm8 */
2733 emulate_push(ctxt, ops);
2735 case 0x6c: /* insb */
2736 case 0x6d: /* insw/insd */
2737 c->dst.bytes = min(c->dst.bytes, 4u);
2738 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2740 emulate_gp(ctxt, 0);
2743 if (!pio_in_emulated(ctxt, ops, c->dst.bytes,
2744 c->regs[VCPU_REGS_RDX], &c->dst.val))
2745 goto done; /* IO is needed, skip writeback */
2747 case 0x6e: /* outsb */
2748 case 0x6f: /* outsw/outsd */
2749 c->src.bytes = min(c->src.bytes, 4u);
2750 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2752 emulate_gp(ctxt, 0);
2755 ops->pio_out_emulated(c->src.bytes, c->regs[VCPU_REGS_RDX],
2756 &c->src.val, 1, ctxt->vcpu);
2758 c->dst.type = OP_NONE; /* nothing to writeback */
2760 case 0x70 ... 0x7f: /* jcc (short) */
2761 if (test_cc(c->b, ctxt->eflags))
2762 jmp_rel(c, c->src.val);
2764 case 0x80 ... 0x83: /* Grp1 */
2765 switch (c->modrm_reg) {
2786 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
2788 case 0x86 ... 0x87: /* xchg */
2790 /* Write back the register source. */
2791 switch (c->dst.bytes) {
2793 *(u8 *) c->src.ptr = (u8) c->dst.val;
2796 *(u16 *) c->src.ptr = (u16) c->dst.val;
2799 *c->src.ptr = (u32) c->dst.val;
2800 break; /* 64b reg: zero-extend */
2802 *c->src.ptr = c->dst.val;
2806 * Write back the memory destination with implicit LOCK
2809 c->dst.val = c->src.val;
2812 case 0x88 ... 0x8b: /* mov */
2814 case 0x8c: /* mov r/m, sreg */
2815 if (c->modrm_reg > VCPU_SREG_GS) {
2819 c->dst.val = ops->get_segment_selector(c->modrm_reg, ctxt->vcpu);
2821 case 0x8d: /* lea r16/r32, m */
2822 c->dst.val = c->modrm_ea;
2824 case 0x8e: { /* mov seg, r/m16 */
2829 if (c->modrm_reg == VCPU_SREG_CS ||
2830 c->modrm_reg > VCPU_SREG_GS) {
2835 if (c->modrm_reg == VCPU_SREG_SS)
2836 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2838 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
2840 c->dst.type = OP_NONE; /* Disable writeback. */
2843 case 0x8f: /* pop (sole member of Grp1a) */
2844 rc = emulate_grp1a(ctxt, ops);
2845 if (rc != X86EMUL_CONTINUE)
2848 case 0x90: /* nop / xchg r8,rax */
2849 if (c->dst.ptr == (unsigned long *)&c->regs[VCPU_REGS_RAX]) {
2850 c->dst.type = OP_NONE; /* nop */
2853 case 0x91 ... 0x97: /* xchg reg,rax */
2854 c->src.type = OP_REG;
2855 c->src.bytes = c->op_bytes;
2856 c->src.ptr = (unsigned long *) &c->regs[VCPU_REGS_RAX];
2857 c->src.val = *(c->src.ptr);
2859 case 0x9c: /* pushf */
2860 c->src.val = (unsigned long) ctxt->eflags;
2861 emulate_push(ctxt, ops);
2863 case 0x9d: /* popf */
2864 c->dst.type = OP_REG;
2865 c->dst.ptr = (unsigned long *) &ctxt->eflags;
2866 c->dst.bytes = c->op_bytes;
2867 rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
2868 if (rc != X86EMUL_CONTINUE)
2871 case 0xa0 ... 0xa3: /* mov */
2872 case 0xa4 ... 0xa5: /* movs */
2874 case 0xa6 ... 0xa7: /* cmps */
2875 c->dst.type = OP_NONE; /* Disable writeback. */
2876 DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.ptr, c->dst.ptr);
2878 case 0xa8 ... 0xa9: /* test ax, imm */
2880 case 0xaa ... 0xab: /* stos */
2881 c->dst.val = c->regs[VCPU_REGS_RAX];
2883 case 0xac ... 0xad: /* lods */
2885 case 0xae ... 0xaf: /* scas */
2886 DPRINTF("Urk! I don't handle SCAS.\n");
2887 goto cannot_emulate;
2888 case 0xb0 ... 0xbf: /* mov r, imm */
2893 case 0xc3: /* ret */
2894 c->dst.type = OP_REG;
2895 c->dst.ptr = &c->eip;
2896 c->dst.bytes = c->op_bytes;
2897 goto pop_instruction;
2898 case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */
2900 c->dst.val = c->src.val;
2902 case 0xcb: /* ret far */
2903 rc = emulate_ret_far(ctxt, ops);
2904 if (rc != X86EMUL_CONTINUE)
2907 case 0xd0 ... 0xd1: /* Grp2 */
2911 case 0xd2 ... 0xd3: /* Grp2 */
2912 c->src.val = c->regs[VCPU_REGS_RCX];
2915 case 0xe4: /* inb */
2918 case 0xe6: /* outb */
2919 case 0xe7: /* out */
2921 case 0xe8: /* call (near) */ {
2922 long int rel = c->src.val;
2923 c->src.val = (unsigned long) c->eip;
2925 emulate_push(ctxt, ops);
2928 case 0xe9: /* jmp rel */
2930 case 0xea: { /* jmp far */
2933 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
2935 if (load_segment_descriptor(ctxt, ops, sel, VCPU_SREG_CS))
2939 memcpy(&c->eip, c->src.valptr, c->op_bytes);
2943 jmp: /* jmp rel short */
2944 jmp_rel(c, c->src.val);
2945 c->dst.type = OP_NONE; /* Disable writeback. */
2947 case 0xec: /* in al,dx */
2948 case 0xed: /* in (e/r)ax,dx */
2949 c->src.val = c->regs[VCPU_REGS_RDX];
2951 c->dst.bytes = min(c->dst.bytes, 4u);
2952 if (!emulator_io_permited(ctxt, ops, c->src.val, c->dst.bytes)) {
2953 emulate_gp(ctxt, 0);
2956 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
2958 goto done; /* IO is needed */
2960 case 0xee: /* out dx,al */
2961 case 0xef: /* out dx,(e/r)ax */
2962 c->src.val = c->regs[VCPU_REGS_RDX];
2964 c->dst.bytes = min(c->dst.bytes, 4u);
2965 if (!emulator_io_permited(ctxt, ops, c->src.val, c->dst.bytes)) {
2966 emulate_gp(ctxt, 0);
2969 ops->pio_out_emulated(c->dst.bytes, c->src.val, &c->dst.val, 1,
2971 c->dst.type = OP_NONE; /* Disable writeback. */
2973 case 0xf4: /* hlt */
2974 ctxt->vcpu->arch.halt_request = 1;
2976 case 0xf5: /* cmc */
2977 /* complement carry flag from eflags reg */
2978 ctxt->eflags ^= EFLG_CF;
2979 c->dst.type = OP_NONE; /* Disable writeback. */
2981 case 0xf6 ... 0xf7: /* Grp3 */
2982 if (!emulate_grp3(ctxt, ops))
2983 goto cannot_emulate;
2985 case 0xf8: /* clc */
2986 ctxt->eflags &= ~EFLG_CF;
2987 c->dst.type = OP_NONE; /* Disable writeback. */
2989 case 0xfa: /* cli */
2990 if (emulator_bad_iopl(ctxt, ops)) {
2991 emulate_gp(ctxt, 0);
2994 ctxt->eflags &= ~X86_EFLAGS_IF;
2995 c->dst.type = OP_NONE; /* Disable writeback. */
2998 case 0xfb: /* sti */
2999 if (emulator_bad_iopl(ctxt, ops)) {
3000 emulate_gp(ctxt, 0);
3003 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3004 ctxt->eflags |= X86_EFLAGS_IF;
3005 c->dst.type = OP_NONE; /* Disable writeback. */
3008 case 0xfc: /* cld */
3009 ctxt->eflags &= ~EFLG_DF;
3010 c->dst.type = OP_NONE; /* Disable writeback. */
3012 case 0xfd: /* std */
3013 ctxt->eflags |= EFLG_DF;
3014 c->dst.type = OP_NONE; /* Disable writeback. */
3016 case 0xfe: /* Grp4 */
3018 rc = emulate_grp45(ctxt, ops);
3019 if (rc != X86EMUL_CONTINUE)
3022 case 0xff: /* Grp5 */
3023 if (c->modrm_reg == 5)
3027 goto cannot_emulate;
3031 rc = writeback(ctxt, ops);
3032 if (rc != X86EMUL_CONTINUE)
3036 * restore dst type in case the decoding will be reused
3037 * (happens for string instruction )
3039 c->dst.type = saved_dst_type;
3041 if ((c->d & SrcMask) == SrcSI)
3042 string_addr_inc(ctxt, seg_override_base(ctxt, ops, c),
3043 VCPU_REGS_RSI, &c->src);
3045 if ((c->d & DstMask) == DstDI)
3046 string_addr_inc(ctxt, es_base(ctxt, ops), VCPU_REGS_RDI,
3049 if (c->rep_prefix && (c->d & String)) {
3050 struct read_cache *rc = &ctxt->decode.io_read;
3051 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3053 * Re-enter guest when pio read ahead buffer is empty or,
3054 * if it is not used, after each 1024 iteration.
3056 if ((rc->end == 0 && !(c->regs[VCPU_REGS_RCX] & 0x3ff)) ||
3057 (rc->end != 0 && rc->end == rc->pos))
3058 ctxt->restart = false;
3061 * reset read cache here in case string instruction is restared
3064 ctxt->decode.mem_read.end = 0;
3068 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
3072 case 0x01: /* lgdt, lidt, lmsw */
3073 switch (c->modrm_reg) {
3075 unsigned long address;
3077 case 0: /* vmcall */
3078 if (c->modrm_mod != 3 || c->modrm_rm != 1)
3079 goto cannot_emulate;
3081 rc = kvm_fix_hypercall(ctxt->vcpu);
3082 if (rc != X86EMUL_CONTINUE)
3085 /* Let the processor re-execute the fixed hypercall */
3087 /* Disable writeback. */
3088 c->dst.type = OP_NONE;
3091 rc = read_descriptor(ctxt, ops, c->src.ptr,
3092 &size, &address, c->op_bytes);
3093 if (rc != X86EMUL_CONTINUE)
3095 realmode_lgdt(ctxt->vcpu, size, address);
3096 /* Disable writeback. */
3097 c->dst.type = OP_NONE;
3099 case 3: /* lidt/vmmcall */
3100 if (c->modrm_mod == 3) {
3101 switch (c->modrm_rm) {
3103 rc = kvm_fix_hypercall(ctxt->vcpu);
3104 if (rc != X86EMUL_CONTINUE)
3108 goto cannot_emulate;
3111 rc = read_descriptor(ctxt, ops, c->src.ptr,
3114 if (rc != X86EMUL_CONTINUE)
3116 realmode_lidt(ctxt->vcpu, size, address);
3118 /* Disable writeback. */
3119 c->dst.type = OP_NONE;
3123 c->dst.val = ops->get_cr(0, ctxt->vcpu);
3126 ops->set_cr(0, (ops->get_cr(0, ctxt->vcpu) & ~0x0ful) |
3127 (c->src.val & 0x0f), ctxt->vcpu);
3128 c->dst.type = OP_NONE;
3130 case 5: /* not defined */
3134 emulate_invlpg(ctxt->vcpu, c->modrm_ea);
3135 /* Disable writeback. */
3136 c->dst.type = OP_NONE;
3139 goto cannot_emulate;
3142 case 0x05: /* syscall */
3143 rc = emulate_syscall(ctxt, ops);
3144 if (rc != X86EMUL_CONTINUE)
3150 emulate_clts(ctxt->vcpu);
3151 c->dst.type = OP_NONE;
3153 case 0x09: /* wbinvd */
3154 kvm_emulate_wbinvd(ctxt->vcpu);
3155 c->dst.type = OP_NONE;
3157 case 0x08: /* invd */
3158 case 0x0d: /* GrpP (prefetch) */
3159 case 0x18: /* Grp16 (prefetch/nop) */
3160 c->dst.type = OP_NONE;
3162 case 0x20: /* mov cr, reg */
3163 switch (c->modrm_reg) {
3170 c->regs[c->modrm_rm] = ops->get_cr(c->modrm_reg, ctxt->vcpu);
3171 c->dst.type = OP_NONE; /* no writeback */
3173 case 0x21: /* mov from dr to reg */
3174 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
3175 (c->modrm_reg == 4 || c->modrm_reg == 5)) {
3179 ops->get_dr(c->modrm_reg, &c->regs[c->modrm_rm], ctxt->vcpu);
3180 c->dst.type = OP_NONE; /* no writeback */
3182 case 0x22: /* mov reg, cr */
3183 if (ops->set_cr(c->modrm_reg, c->modrm_val, ctxt->vcpu)) {
3184 emulate_gp(ctxt, 0);
3187 c->dst.type = OP_NONE;
3189 case 0x23: /* mov from reg to dr */
3190 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
3191 (c->modrm_reg == 4 || c->modrm_reg == 5)) {
3196 if (ops->set_dr(c->modrm_reg, c->regs[c->modrm_rm] &
3197 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
3198 ~0ULL : ~0U), ctxt->vcpu) < 0) {
3199 /* #UD condition is already handled by the code above */
3200 emulate_gp(ctxt, 0);
3204 c->dst.type = OP_NONE; /* no writeback */
3208 msr_data = (u32)c->regs[VCPU_REGS_RAX]
3209 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
3210 if (ops->set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) {
3211 emulate_gp(ctxt, 0);
3214 rc = X86EMUL_CONTINUE;
3215 c->dst.type = OP_NONE;
3219 if (ops->get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) {
3220 emulate_gp(ctxt, 0);
3223 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
3224 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
3226 rc = X86EMUL_CONTINUE;
3227 c->dst.type = OP_NONE;
3229 case 0x34: /* sysenter */
3230 rc = emulate_sysenter(ctxt, ops);
3231 if (rc != X86EMUL_CONTINUE)
3236 case 0x35: /* sysexit */
3237 rc = emulate_sysexit(ctxt, ops);
3238 if (rc != X86EMUL_CONTINUE)
3243 case 0x40 ... 0x4f: /* cmov */
3244 c->dst.val = c->dst.orig_val = c->src.val;
3245 if (!test_cc(c->b, ctxt->eflags))
3246 c->dst.type = OP_NONE; /* no writeback */
3248 case 0x80 ... 0x8f: /* jnz rel, etc*/
3249 if (test_cc(c->b, ctxt->eflags))
3250 jmp_rel(c, c->src.val);
3251 c->dst.type = OP_NONE;
3253 case 0xa0: /* push fs */
3254 emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
3256 case 0xa1: /* pop fs */
3257 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
3258 if (rc != X86EMUL_CONTINUE)
3263 c->dst.type = OP_NONE;
3264 /* only subword offset */
3265 c->src.val &= (c->dst.bytes << 3) - 1;
3266 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
3268 case 0xa4: /* shld imm8, r, r/m */
3269 case 0xa5: /* shld cl, r, r/m */
3270 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
3272 case 0xa8: /* push gs */
3273 emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
3275 case 0xa9: /* pop gs */
3276 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
3277 if (rc != X86EMUL_CONTINUE)
3282 /* only subword offset */
3283 c->src.val &= (c->dst.bytes << 3) - 1;
3284 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
3286 case 0xac: /* shrd imm8, r, r/m */
3287 case 0xad: /* shrd cl, r, r/m */
3288 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
3290 case 0xae: /* clflush */
3292 case 0xb0 ... 0xb1: /* cmpxchg */
3294 * Save real source value, then compare EAX against
3297 c->src.orig_val = c->src.val;
3298 c->src.val = c->regs[VCPU_REGS_RAX];
3299 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
3300 if (ctxt->eflags & EFLG_ZF) {
3301 /* Success: write back to memory. */
3302 c->dst.val = c->src.orig_val;
3304 /* Failure: write the value we saw to EAX. */
3305 c->dst.type = OP_REG;
3306 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
3311 /* only subword offset */
3312 c->src.val &= (c->dst.bytes << 3) - 1;
3313 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
3315 case 0xb6 ... 0xb7: /* movzx */
3316 c->dst.bytes = c->op_bytes;
3317 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
3320 case 0xba: /* Grp8 */
3321 switch (c->modrm_reg & 3) {
3334 /* only subword offset */
3335 c->src.val &= (c->dst.bytes << 3) - 1;
3336 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
3338 case 0xbe ... 0xbf: /* movsx */
3339 c->dst.bytes = c->op_bytes;
3340 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
3343 case 0xc3: /* movnti */
3344 c->dst.bytes = c->op_bytes;
3345 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
3348 case 0xc7: /* Grp9 (cmpxchg8b) */
3349 rc = emulate_grp9(ctxt, ops);
3350 if (rc != X86EMUL_CONTINUE)
3354 goto cannot_emulate;
3359 DPRINTF("Cannot emulate %02x\n", c->b);