Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/hch/hfsplus
[pandora-kernel.git] / arch / x86 / kvm / emulate.c
index 8e12e1b..38b6e8d 100644 (file)
@@ -9,7 +9,7 @@
  * privileged instructions:
  *
  * Copyright (C) 2006 Qumranet
- * Copyright 2010 Red Hat, Inc. and/or its affilates.
+ * Copyright 2010 Red Hat, Inc. and/or its affiliates.
  *
  *   Avi Kivity <avi@qumranet.com>
  *   Yaniv Kamay <yaniv@qumranet.com>
@@ -58,7 +58,6 @@
 #define DstMask     (7<<1)
 /* Source operand type. */
 #define SrcNone     (0<<4)     /* No source operand. */
-#define SrcImplicit (0<<4)     /* Source operand is implicit in the opcode. */
 #define SrcReg      (1<<4)     /* Register operand. */
 #define SrcMem      (2<<4)     /* Memory operand. */
 #define SrcMem16    (3<<4)     /* Memory operand (16-bit). */
@@ -72,6 +71,7 @@
 #define SrcImmFAddr (0xb<<4)   /* Source is immediate far address */
 #define SrcMemFAddr (0xc<<4)   /* Source is far address in memory */
 #define SrcAcc      (0xd<<4)   /* Source Accumulator */
+#define SrcImmU16   (0xe<<4)    /* Immediate operand, unsigned, 16 bits */
 #define SrcMask     (0xf<<4)
 /* Generic ModRM decode. */
 #define ModRM       (1<<8)
@@ -95,6 +95,7 @@
 #define Src2CL      (1<<29)
 #define Src2ImmByte (2<<29)
 #define Src2One     (3<<29)
+#define Src2Imm     (4<<29)
 #define Src2Mask    (7<<29)
 
 #define X2(x...) x, x
@@ -194,13 +195,13 @@ struct group_dual {
 #define ON64(x)
 #endif
 
-#define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix)     \
+#define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix, _dsttype) \
        do {                                                            \
                __asm__ __volatile__ (                                  \
                        _PRE_EFLAGS("0", "4", "2")                      \
                        _op _suffix " %"_x"3,%1; "                      \
                        _POST_EFLAGS("0", "4", "2")                     \
-                       : "=m" (_eflags), "=m" ((_dst).val),            \
+                       : "=m" (_eflags), "+q" (*(_dsttype*)&(_dst).val),\
                          "=&r" (_tmp)                                  \
                        : _y ((_src).val), "i" (EFLAGS_MASK));          \
        } while (0)
@@ -213,13 +214,13 @@ struct group_dual {
                                                                        \
                switch ((_dst).bytes) {                                 \
                case 2:                                                 \
-                       ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w"); \
+                       ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w",u16);\
                        break;                                          \
                case 4:                                                 \
-                       ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l"); \
+                       ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l",u32);\
                        break;                                          \
                case 8:                                                 \
-                       ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q")); \
+                       ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q",u64)); \
                        break;                                          \
                }                                                       \
        } while (0)
@@ -229,7 +230,7 @@ struct group_dual {
                unsigned long _tmp;                                          \
                switch ((_dst).bytes) {                                      \
                case 1:                                                      \
-                       ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b");  \
+                       ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b",u8); \
                        break;                                               \
                default:                                                     \
                        __emulate_2op_nobyte(_op, _src, _dst, _eflags,       \
@@ -330,6 +331,27 @@ struct group_dual {
                          "a" (_rax), "d" (_rdx));                      \
        } while (0)
 
+#define __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _suffix, _ex) \
+       do {                                                            \
+               unsigned long _tmp;                                     \
+                                                                       \
+               __asm__ __volatile__ (                                  \
+                       _PRE_EFLAGS("0", "5", "1")                      \
+                       "1: \n\t"                                       \
+                       _op _suffix " %6; "                             \
+                       "2: \n\t"                                       \
+                       _POST_EFLAGS("0", "5", "1")                     \
+                       ".pushsection .fixup,\"ax\" \n\t"               \
+                       "3: movb $1, %4 \n\t"                           \
+                       "jmp 2b \n\t"                                   \
+                       ".popsection \n\t"                              \
+                       _ASM_EXTABLE(1b, 3b)                            \
+                       : "=m" (_eflags), "=&r" (_tmp),                 \
+                         "+a" (_rax), "+d" (_rdx), "+qm"(_ex)          \
+                       : "i" (EFLAGS_MASK), "m" ((_src).val),          \
+                         "a" (_rax), "d" (_rdx));                      \
+       } while (0)
+
 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
 #define emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags)                    \
        do {                                                                    \
@@ -341,6 +363,28 @@ struct group_dual {
                }                                                       \
        } while (0)
 
+#define emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _ex)    \
+       do {                                                            \
+               switch((_src).bytes) {                                  \
+               case 1:                                                 \
+                       __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
+                                                _eflags, "b", _ex);    \
+                       break;                                          \
+               case 2:                                                 \
+                       __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
+                                                _eflags, "w", _ex);    \
+                       break;                                          \
+               case 4:                                                 \
+                       __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
+                                                _eflags, "l", _ex);    \
+                       break;                                          \
+               case 8: ON64(                                           \
+                       __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
+                                                _eflags, "q", _ex));   \
+                       break;                                          \
+               }                                                       \
+       } while (0)
+
 /* Fetch next part of the instruction being emulated. */
 #define insn_fetch(_type, _size, _eip)                                  \
 ({     unsigned long _x;                                               \
@@ -436,7 +480,6 @@ static void emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
        ctxt->exception = vec;
        ctxt->error_code = error;
        ctxt->error_code_valid = valid;
-       ctxt->restart = false;
 }
 
 static void emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
@@ -444,11 +487,9 @@ static void emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
        emulate_exception(ctxt, GP_VECTOR, err, true);
 }
 
-static void emulate_pf(struct x86_emulate_ctxt *ctxt, unsigned long addr,
-                      int err)
+static void emulate_pf(struct x86_emulate_ctxt *ctxt)
 {
-       ctxt->cr2 = addr;
-       emulate_exception(ctxt, PF_VECTOR, err, true);
+       emulate_exception(ctxt, PF_VECTOR, 0, true);
 }
 
 static void emulate_ud(struct x86_emulate_ctxt *ctxt)
@@ -461,6 +502,12 @@ static void emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
        emulate_exception(ctxt, TS_VECTOR, err, true);
 }
 
+static int emulate_de(struct x86_emulate_ctxt *ctxt)
+{
+       emulate_exception(ctxt, DE_VECTOR, 0, false);
+       return X86EMUL_PROPAGATE_FAULT;
+}
+
 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
                              struct x86_emulate_ops *ops,
                              unsigned long eip, u8 *dest)
@@ -751,7 +798,7 @@ done:
 
 static void fetch_bit_operand(struct decode_cache *c)
 {
-       long sv, mask;
+       long sv = 0, mask;
 
        if (c->dst.type == OP_MEM && c->src.type == OP_REG) {
                mask = ~(c->dst.bytes * 8 - 1);
@@ -785,7 +832,7 @@ static int read_emulated(struct x86_emulate_ctxt *ctxt,
                rc = ops->read_emulated(addr, mc->data + mc->end, n, &err,
                                        ctxt->vcpu);
                if (rc == X86EMUL_PROPAGATE_FAULT)
-                       emulate_pf(ctxt, addr, err);
+                       emulate_pf(ctxt);
                if (rc != X86EMUL_CONTINUE)
                        return rc;
                mc->end += n;
@@ -872,7 +919,7 @@ static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
        addr = dt.address + index * 8;
        ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu,  &err);
        if (ret == X86EMUL_PROPAGATE_FAULT)
-               emulate_pf(ctxt, addr, err);
+               emulate_pf(ctxt);
 
        return ret;
 }
@@ -898,7 +945,7 @@ static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
        addr = dt.address + index * 8;
        ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu, &err);
        if (ret == X86EMUL_PROPAGATE_FAULT)
-               emulate_pf(ctxt, addr, err);
+               emulate_pf(ctxt);
 
        return ret;
 }
@@ -1068,7 +1115,7 @@ static inline int writeback(struct x86_emulate_ctxt *ctxt,
                                        &err,
                                        ctxt->vcpu);
                if (rc == X86EMUL_PROPAGATE_FAULT)
-                       emulate_pf(ctxt, c->dst.addr.mem, err);
+                       emulate_pf(ctxt);
                if (rc != X86EMUL_CONTINUE)
                        return rc;
                break;
@@ -1151,6 +1198,9 @@ static int emulate_popf(struct x86_emulate_ctxt *ctxt,
        *(unsigned long *)dest =
                (ctxt->eflags & ~change_mask) | (val & change_mask);
 
+       if (rc == X86EMUL_PROPAGATE_FAULT)
+               emulate_pf(ctxt);
+
        return rc;
 }
 
@@ -1232,7 +1282,7 @@ int emulate_int_real(struct x86_emulate_ctxt *ctxt,
                               struct x86_emulate_ops *ops, int irq)
 {
        struct decode_cache *c = &ctxt->decode;
-       int rc = X86EMUL_CONTINUE;
+       int rc;
        struct desc_ptr dt;
        gva_t cs_addr;
        gva_t eip_addr;
@@ -1242,14 +1292,25 @@ int emulate_int_real(struct x86_emulate_ctxt *ctxt,
        /* TODO: Add limit checks */
        c->src.val = ctxt->eflags;
        emulate_push(ctxt, ops);
+       rc = writeback(ctxt, ops);
+       if (rc != X86EMUL_CONTINUE)
+               return rc;
 
        ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
 
        c->src.val = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
        emulate_push(ctxt, ops);
+       rc = writeback(ctxt, ops);
+       if (rc != X86EMUL_CONTINUE)
+               return rc;
 
        c->src.val = c->eip;
        emulate_push(ctxt, ops);
+       rc = writeback(ctxt, ops);
+       if (rc != X86EMUL_CONTINUE)
+               return rc;
+
+       c->dst.type = OP_NONE;
 
        ops->get_idt(&dt, ctxt->vcpu);
 
@@ -1404,6 +1465,7 @@ static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
        struct decode_cache *c = &ctxt->decode;
        unsigned long *rax = &c->regs[VCPU_REGS_RAX];
        unsigned long *rdx = &c->regs[VCPU_REGS_RDX];
+       u8 de = 0;
 
        switch (c->modrm_reg) {
        case 0 ... 1:   /* test */
@@ -1422,14 +1484,18 @@ static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
                emulate_1op_rax_rdx("imul", c->src, *rax, *rdx, ctxt->eflags);
                break;
        case 6: /* div */
-               emulate_1op_rax_rdx("div", c->src, *rax, *rdx, ctxt->eflags);
+               emulate_1op_rax_rdx_ex("div", c->src, *rax, *rdx,
+                                      ctxt->eflags, de);
                break;
        case 7: /* idiv */
-               emulate_1op_rax_rdx("idiv", c->src, *rax, *rdx, ctxt->eflags);
+               emulate_1op_rax_rdx_ex("idiv", c->src, *rax, *rdx,
+                                      ctxt->eflags, de);
                break;
        default:
                return X86EMUL_UNHANDLEABLE;
        }
+       if (de)
+               return emulate_de(ctxt);
        return X86EMUL_CONTINUE;
 }
 
@@ -1502,6 +1568,23 @@ static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
        return rc;
 }
 
+static int emulate_load_segment(struct x86_emulate_ctxt *ctxt,
+                          struct x86_emulate_ops *ops, int seg)
+{
+       struct decode_cache *c = &ctxt->decode;
+       unsigned short sel;
+       int rc;
+
+       memcpy(&sel, c->src.valptr + c->op_bytes, 2);
+
+       rc = load_segment_descriptor(ctxt, ops, sel, seg);
+       if (rc != X86EMUL_CONTINUE)
+               return rc;
+
+       c->dst.val = c->src.val;
+       return rc;
+}
+
 static inline void
 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
                        struct x86_emulate_ops *ops, struct desc_struct *cs,
@@ -1857,7 +1940,7 @@ static int task_switch_16(struct x86_emulate_ctxt *ctxt,
                            &err);
        if (ret == X86EMUL_PROPAGATE_FAULT) {
                /* FIXME: need to provide precise fault address */
-               emulate_pf(ctxt, old_tss_base, err);
+               emulate_pf(ctxt);
                return ret;
        }
 
@@ -1867,7 +1950,7 @@ static int task_switch_16(struct x86_emulate_ctxt *ctxt,
                             &err);
        if (ret == X86EMUL_PROPAGATE_FAULT) {
                /* FIXME: need to provide precise fault address */
-               emulate_pf(ctxt, old_tss_base, err);
+               emulate_pf(ctxt);
                return ret;
        }
 
@@ -1875,7 +1958,7 @@ static int task_switch_16(struct x86_emulate_ctxt *ctxt,
                            &err);
        if (ret == X86EMUL_PROPAGATE_FAULT) {
                /* FIXME: need to provide precise fault address */
-               emulate_pf(ctxt, new_tss_base, err);
+               emulate_pf(ctxt);
                return ret;
        }
 
@@ -1888,7 +1971,7 @@ static int task_switch_16(struct x86_emulate_ctxt *ctxt,
                                     ctxt->vcpu, &err);
                if (ret == X86EMUL_PROPAGATE_FAULT) {
                        /* FIXME: need to provide precise fault address */
-                       emulate_pf(ctxt, new_tss_base, err);
+                       emulate_pf(ctxt);
                        return ret;
                }
        }
@@ -1999,7 +2082,7 @@ static int task_switch_32(struct x86_emulate_ctxt *ctxt,
                            &err);
        if (ret == X86EMUL_PROPAGATE_FAULT) {
                /* FIXME: need to provide precise fault address */
-               emulate_pf(ctxt, old_tss_base, err);
+               emulate_pf(ctxt);
                return ret;
        }
 
@@ -2009,7 +2092,7 @@ static int task_switch_32(struct x86_emulate_ctxt *ctxt,
                             &err);
        if (ret == X86EMUL_PROPAGATE_FAULT) {
                /* FIXME: need to provide precise fault address */
-               emulate_pf(ctxt, old_tss_base, err);
+               emulate_pf(ctxt);
                return ret;
        }
 
@@ -2017,7 +2100,7 @@ static int task_switch_32(struct x86_emulate_ctxt *ctxt,
                            &err);
        if (ret == X86EMUL_PROPAGATE_FAULT) {
                /* FIXME: need to provide precise fault address */
-               emulate_pf(ctxt, new_tss_base, err);
+               emulate_pf(ctxt);
                return ret;
        }
 
@@ -2030,7 +2113,7 @@ static int task_switch_32(struct x86_emulate_ctxt *ctxt,
                                     ctxt->vcpu, &err);
                if (ret == X86EMUL_PROPAGATE_FAULT) {
                        /* FIXME: need to provide precise fault address */
-                       emulate_pf(ctxt, new_tss_base, err);
+                       emulate_pf(ctxt);
                        return ret;
                }
        }
@@ -2164,12 +2247,159 @@ static int em_push(struct x86_emulate_ctxt *ctxt)
        return X86EMUL_CONTINUE;
 }
 
+static int em_das(struct x86_emulate_ctxt *ctxt)
+{
+       struct decode_cache *c = &ctxt->decode;
+       u8 al, old_al;
+       bool af, cf, old_cf;
+
+       cf = ctxt->eflags & X86_EFLAGS_CF;
+       al = c->dst.val;
+
+       old_al = al;
+       old_cf = cf;
+       cf = false;
+       af = ctxt->eflags & X86_EFLAGS_AF;
+       if ((al & 0x0f) > 9 || af) {
+               al -= 6;
+               cf = old_cf | (al >= 250);
+               af = true;
+       } else {
+               af = false;
+       }
+       if (old_al > 0x99 || old_cf) {
+               al -= 0x60;
+               cf = true;
+       }
+
+       c->dst.val = al;
+       /* Set PF, ZF, SF */
+       c->src.type = OP_IMM;
+       c->src.val = 0;
+       c->src.bytes = 1;
+       emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
+       ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
+       if (cf)
+               ctxt->eflags |= X86_EFLAGS_CF;
+       if (af)
+               ctxt->eflags |= X86_EFLAGS_AF;
+       return X86EMUL_CONTINUE;
+}
+
+static int em_call_far(struct x86_emulate_ctxt *ctxt)
+{
+       struct decode_cache *c = &ctxt->decode;
+       u16 sel, old_cs;
+       ulong old_eip;
+       int rc;
+
+       old_cs = ctxt->ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
+       old_eip = c->eip;
+
+       memcpy(&sel, c->src.valptr + c->op_bytes, 2);
+       if (load_segment_descriptor(ctxt, ctxt->ops, sel, VCPU_SREG_CS))
+               return X86EMUL_CONTINUE;
+
+       c->eip = 0;
+       memcpy(&c->eip, c->src.valptr, c->op_bytes);
+
+       c->src.val = old_cs;
+       emulate_push(ctxt, ctxt->ops);
+       rc = writeback(ctxt, ctxt->ops);
+       if (rc != X86EMUL_CONTINUE)
+               return rc;
+
+       c->src.val = old_eip;
+       emulate_push(ctxt, ctxt->ops);
+       rc = writeback(ctxt, ctxt->ops);
+       if (rc != X86EMUL_CONTINUE)
+               return rc;
+
+       c->dst.type = OP_NONE;
+
+       return X86EMUL_CONTINUE;
+}
+
+static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
+{
+       struct decode_cache *c = &ctxt->decode;
+       int rc;
+
+       c->dst.type = OP_REG;
+       c->dst.addr.reg = &c->eip;
+       c->dst.bytes = c->op_bytes;
+       rc = emulate_pop(ctxt, ctxt->ops, &c->dst.val, c->op_bytes);
+       if (rc != X86EMUL_CONTINUE)
+               return rc;
+       register_address_increment(c, &c->regs[VCPU_REGS_RSP], c->src.val);
+       return X86EMUL_CONTINUE;
+}
+
+static int em_imul(struct x86_emulate_ctxt *ctxt)
+{
+       struct decode_cache *c = &ctxt->decode;
+
+       emulate_2op_SrcV_nobyte("imul", c->src, c->dst, ctxt->eflags);
+       return X86EMUL_CONTINUE;
+}
+
+static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
+{
+       struct decode_cache *c = &ctxt->decode;
+
+       c->dst.val = c->src2.val;
+       return em_imul(ctxt);
+}
+
+static int em_cwd(struct x86_emulate_ctxt *ctxt)
+{
+       struct decode_cache *c = &ctxt->decode;
+
+       c->dst.type = OP_REG;
+       c->dst.bytes = c->src.bytes;
+       c->dst.addr.reg = &c->regs[VCPU_REGS_RDX];
+       c->dst.val = ~((c->src.val >> (c->src.bytes * 8 - 1)) - 1);
+
+       return X86EMUL_CONTINUE;
+}
+
+static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
+{
+       unsigned cpl = ctxt->ops->cpl(ctxt->vcpu);
+       struct decode_cache *c = &ctxt->decode;
+       u64 tsc = 0;
+
+       if (cpl > 0 && (ctxt->ops->get_cr(4, ctxt->vcpu) & X86_CR4_TSD)) {
+               emulate_gp(ctxt, 0);
+               return X86EMUL_PROPAGATE_FAULT;
+       }
+       ctxt->ops->get_msr(ctxt->vcpu, MSR_IA32_TSC, &tsc);
+       c->regs[VCPU_REGS_RAX] = (u32)tsc;
+       c->regs[VCPU_REGS_RDX] = tsc >> 32;
+       return X86EMUL_CONTINUE;
+}
+
+static int em_mov(struct x86_emulate_ctxt *ctxt)
+{
+       struct decode_cache *c = &ctxt->decode;
+       c->dst.val = c->src.val;
+       return X86EMUL_CONTINUE;
+}
+
 #define D(_y) { .flags = (_y) }
 #define N    D(0)
 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
 #define GD(_f, _g) { .flags = ((_f) | Group | GroupDual), .u.gdual = (_g) }
 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
 
+#define D2bv(_f)      D((_f) | ByteOp), D(_f)
+#define I2bv(_f, _e)  I((_f) | ByteOp, _e), I(_f, _e)
+
+#define D6ALU(_f) D2bv((_f) | DstMem | SrcReg | ModRM),                        \
+               D2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock),         \
+               D2bv(((_f) & ~Lock) | DstAcc | SrcImm)
+
+
 static struct opcode group1[] = {
        X7(D(Lock)), N
 };
@@ -2191,7 +2421,8 @@ static struct opcode group4[] = {
 
 static struct opcode group5[] = {
        D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
-       D(SrcMem | ModRM | Stack), N,
+       D(SrcMem | ModRM | Stack),
+       I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
        D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
        D(SrcMem | ModRM | Stack), N,
 };
@@ -2219,44 +2450,31 @@ static struct group_dual group9 = { {
        N, N, N, N, N, N, N, N,
 } };
 
+static struct opcode group11[] = {
+       I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
+};
+
 static struct opcode opcode_table[256] = {
        /* 0x00 - 0x07 */
-       D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
-       D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
-       D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
+       D6ALU(Lock),
        D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
        /* 0x08 - 0x0F */
-       D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
-       D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
-       D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
+       D6ALU(Lock),
        D(ImplicitOps | Stack | No64), N,
        /* 0x10 - 0x17 */
-       D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
-       D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
-       D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
+       D6ALU(Lock),
        D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
        /* 0x18 - 0x1F */
-       D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
-       D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
-       D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
+       D6ALU(Lock),
        D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
        /* 0x20 - 0x27 */
-       D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
-       D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
-       D(ByteOp | DstAcc | SrcImmByte), D(DstAcc | SrcImm), N, N,
+       D6ALU(Lock), N, N,
        /* 0x28 - 0x2F */
-       D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
-       D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
-       D(ByteOp | DstAcc | SrcImmByte), D(DstAcc | SrcImm), N, N,
+       D6ALU(Lock), N, I(ByteOp | DstAcc | No64, em_das),
        /* 0x30 - 0x37 */
-       D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
-       D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
-       D(ByteOp | DstAcc | SrcImmByte), D(DstAcc | SrcImm), N, N,
+       D6ALU(Lock), N, N,
        /* 0x38 - 0x3F */
-       D(ByteOp | DstMem | SrcReg | ModRM), D(DstMem | SrcReg | ModRM),
-       D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
-       D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
-       N, N,
+       D6ALU(0), N, N,
        /* 0x40 - 0x4F */
        X16(D(DstReg)),
        /* 0x50 - 0x57 */
@@ -2268,10 +2486,12 @@ static struct opcode opcode_table[256] = {
        N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
        N, N, N, N,
        /* 0x68 - 0x6F */
-       I(SrcImm | Mov | Stack, em_push), N,
-       I(SrcImmByte | Mov | Stack, em_push), N,
-       D(DstDI | ByteOp | Mov | String), D(DstDI | Mov | String), /* insb, insw/insd */
-       D(SrcSI | ByteOp | ImplicitOps | String), D(SrcSI | ImplicitOps | String), /* outsb, outsw/outsd */
+       I(SrcImm | Mov | Stack, em_push),
+       I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
+       I(SrcImmByte | Mov | Stack, em_push),
+       I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
+       D2bv(DstDI | Mov | String), /* insb, insw/insd */
+       D2bv(SrcSI | ImplicitOps | String), /* outsb, outsw/outsd */
        /* 0x70 - 0x7F */
        X16(D(SrcImmByte)),
        /* 0x80 - 0x87 */
@@ -2279,54 +2499,53 @@ static struct opcode opcode_table[256] = {
        G(DstMem | SrcImm | ModRM | Group, group1),
        G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
        G(DstMem | SrcImmByte | ModRM | Group, group1),
-       D(ByteOp | DstMem | SrcReg | ModRM), D(DstMem | SrcReg | ModRM),
-       D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
+       D2bv(DstMem | SrcReg | ModRM), D2bv(DstMem | SrcReg | ModRM | Lock),
        /* 0x88 - 0x8F */
-       D(ByteOp | DstMem | SrcReg | ModRM | Mov), D(DstMem | SrcReg | ModRM | Mov),
-       D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem | ModRM | Mov),
+       I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
+       I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
        D(DstMem | SrcNone | ModRM | Mov), D(ModRM | SrcMem | NoAccess | DstReg),
        D(ImplicitOps | SrcMem16 | ModRM), G(0, group1A),
        /* 0x90 - 0x97 */
        X8(D(SrcAcc | DstReg)),
        /* 0x98 - 0x9F */
-       N, N, D(SrcImmFAddr | No64), N,
+       D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
+       I(SrcImmFAddr | No64, em_call_far), N,
        D(ImplicitOps | Stack), D(ImplicitOps | Stack), N, N,
        /* 0xA0 - 0xA7 */
-       D(ByteOp | DstAcc | SrcMem | Mov | MemAbs), D(DstAcc | SrcMem | Mov | MemAbs),
-       D(ByteOp | DstMem | SrcAcc | Mov | MemAbs), D(DstMem | SrcAcc | Mov | MemAbs),
-       D(ByteOp | SrcSI | DstDI | Mov | String), D(SrcSI | DstDI | Mov | String),
-       D(ByteOp | SrcSI | DstDI | String), D(SrcSI | DstDI | String),
+       I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
+       I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
+       I2bv(SrcSI | DstDI | Mov | String, em_mov),
+       D2bv(SrcSI | DstDI | String),
        /* 0xA8 - 0xAF */
-       D(DstAcc | SrcImmByte | ByteOp), D(DstAcc | SrcImm),
-       D(ByteOp | SrcAcc | DstDI | Mov | String), D(SrcAcc | DstDI | Mov | String),
-       D(ByteOp | SrcSI | DstAcc | Mov | String), D(SrcSI | DstAcc | Mov | String),
-       D(ByteOp | DstDI | String), D(DstDI | String),
+       D2bv(DstAcc | SrcImm),
+       I2bv(SrcAcc | DstDI | Mov | String, em_mov),
+       I2bv(SrcSI | DstAcc | Mov | String, em_mov),
+       D2bv(SrcAcc | DstDI | String),
        /* 0xB0 - 0xB7 */
-       X8(D(ByteOp | DstReg | SrcImm | Mov)),
+       X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
        /* 0xB8 - 0xBF */
-       X8(D(DstReg | SrcImm | Mov)),
+       X8(I(DstReg | SrcImm | Mov, em_mov)),
        /* 0xC0 - 0xC7 */
-       D(ByteOp | DstMem | SrcImm | ModRM), D(DstMem | SrcImmByte | ModRM),
-       N, D(ImplicitOps | Stack), N, N,
-       D(ByteOp | DstMem | SrcImm | ModRM | Mov), D(DstMem | SrcImm | ModRM | Mov),
+       D2bv(DstMem | SrcImmByte | ModRM),
+       I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
+       D(ImplicitOps | Stack),
+       D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
+       G(ByteOp, group11), G(0, group11),
        /* 0xC8 - 0xCF */
        N, N, N, D(ImplicitOps | Stack),
        D(ImplicitOps), D(SrcImmByte), D(ImplicitOps | No64), D(ImplicitOps),
        /* 0xD0 - 0xD7 */
-       D(ByteOp | DstMem | SrcOne | ModRM), D(DstMem | SrcOne | ModRM),
-       D(ByteOp | DstMem | SrcImplicit | ModRM), D(DstMem | SrcImplicit | ModRM),
+       D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
        N, N, N, N,
        /* 0xD8 - 0xDF */
        N, N, N, N, N, N, N, N,
        /* 0xE0 - 0xE7 */
-       N, N, N, N,
-       D(ByteOp | SrcImmUByte | DstAcc), D(SrcImmUByte | DstAcc),
-       D(ByteOp | SrcAcc | DstImmUByte), D(SrcAcc | DstImmUByte),
+       X4(D(SrcImmByte)),
+       D2bv(SrcImmUByte | DstAcc), D2bv(SrcAcc | DstImmUByte),
        /* 0xE8 - 0xEF */
        D(SrcImm | Stack), D(SrcImm | ImplicitOps),
        D(SrcImmFAddr | No64), D(SrcImmByte | ImplicitOps),
-       D(SrcNone | ByteOp | DstAcc), D(SrcNone | DstAcc),
-       D(ByteOp | SrcAcc | ImplicitOps), D(SrcAcc | ImplicitOps),
+       D2bv(SrcNone | DstAcc), D2bv(SrcAcc | ImplicitOps),
        /* 0xF0 - 0xF7 */
        N, N, N, N,
        D(ImplicitOps | Priv), D(ImplicitOps), G(ByteOp, group3), G(0, group3),
@@ -2349,7 +2568,8 @@ static struct opcode twobyte_table[256] = {
        N, N, N, N,
        N, N, N, N, N, N, N, N,
        /* 0x30 - 0x3F */
-       D(ImplicitOps | Priv), N, D(ImplicitOps | Priv), N,
+       D(ImplicitOps | Priv), I(ImplicitOps, em_rdtsc),
+       D(ImplicitOps | Priv), N,
        D(ImplicitOps), D(ImplicitOps | Priv), N, N,
        N, N, N, N, N, N, N, N,
        /* 0x40 - 0x4F */
@@ -2374,19 +2594,19 @@ static struct opcode twobyte_table[256] = {
        N, D(DstMem | SrcReg | ModRM | BitOp | Lock),
        D(DstMem | SrcReg | Src2ImmByte | ModRM),
        D(DstMem | SrcReg | Src2CL | ModRM),
-       D(ModRM), N,
+       D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
        /* 0xB0 - 0xB7 */
-       D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
-       N, D(DstMem | SrcReg | ModRM | BitOp | Lock),
-       N, N, D(ByteOp | DstReg | SrcMem | ModRM | Mov),
-           D(DstReg | SrcMem16 | ModRM | Mov),
+       D2bv(DstMem | SrcReg | ModRM | Lock),
+       D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
+       D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
+       D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
        /* 0xB8 - 0xBF */
        N, N,
        G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
        D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
        D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
        /* 0xC0 - 0xCF */
-       D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
+       D2bv(DstMem | SrcReg | ModRM | Lock),
        N, D(DstMem | SrcReg | ModRM | Mov),
        N, N, N, GD(0, &group9),
        N, N, N, N, N, N, N, N,
@@ -2404,6 +2624,59 @@ static struct opcode twobyte_table[256] = {
 #undef GD
 #undef I
 
+#undef D2bv
+#undef I2bv
+#undef D6ALU
+
+static unsigned imm_size(struct decode_cache *c)
+{
+       unsigned size;
+
+       size = (c->d & ByteOp) ? 1 : c->op_bytes;
+       if (size == 8)
+               size = 4;
+       return size;
+}
+
+static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
+                     unsigned size, bool sign_extension)
+{
+       struct decode_cache *c = &ctxt->decode;
+       struct x86_emulate_ops *ops = ctxt->ops;
+       int rc = X86EMUL_CONTINUE;
+
+       op->type = OP_IMM;
+       op->bytes = size;
+       op->addr.mem = c->eip;
+       /* NB. Immediates are sign-extended as necessary. */
+       switch (op->bytes) {
+       case 1:
+               op->val = insn_fetch(s8, 1, c->eip);
+               break;
+       case 2:
+               op->val = insn_fetch(s16, 2, c->eip);
+               break;
+       case 4:
+               op->val = insn_fetch(s32, 4, c->eip);
+               break;
+       }
+       if (!sign_extension) {
+               switch (op->bytes) {
+               case 1:
+                       op->val &= 0xff;
+                       break;
+               case 2:
+                       op->val &= 0xffff;
+                       break;
+               case 4:
+                       op->val &= 0xffffffff;
+                       break;
+               }
+       }
+done:
+       return rc;
+}
+
 int
 x86_decode_insn(struct x86_emulate_ctxt *ctxt)
 {
@@ -2415,9 +2688,6 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt)
        struct opcode opcode, *g_mod012, *g_mod3;
        struct operand memop = { .type = OP_NONE };
 
-       /* we cannot decode insn before we complete previous rep insn */
-       WARN_ON(ctxt->restart);
-
        c->eip = ctxt->eip;
        c->fetch.start = c->fetch.end = c->eip;
        ctxt->cs_base = seg_base(ctxt, ops, VCPU_SREG_CS);
@@ -2592,48 +2862,20 @@ done_prefixes:
        srcmem_common:
                c->src = memop;
                break;
+       case SrcImmU16:
+               rc = decode_imm(ctxt, &c->src, 2, false);
+               break;
        case SrcImm:
+               rc = decode_imm(ctxt, &c->src, imm_size(c), true);
+               break;
        case SrcImmU:
-               c->src.type = OP_IMM;
-               c->src.addr.mem = c->eip;
-               c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
-               if (c->src.bytes == 8)
-                       c->src.bytes = 4;
-               /* NB. Immediates are sign-extended as necessary. */
-               switch (c->src.bytes) {
-               case 1:
-                       c->src.val = insn_fetch(s8, 1, c->eip);
-                       break;
-               case 2:
-                       c->src.val = insn_fetch(s16, 2, c->eip);
-                       break;
-               case 4:
-                       c->src.val = insn_fetch(s32, 4, c->eip);
-                       break;
-               }
-               if ((c->d & SrcMask) == SrcImmU) {
-                       switch (c->src.bytes) {
-                       case 1:
-                               c->src.val &= 0xff;
-                               break;
-                       case 2:
-                               c->src.val &= 0xffff;
-                               break;
-                       case 4:
-                               c->src.val &= 0xffffffff;
-                               break;
-                       }
-               }
+               rc = decode_imm(ctxt, &c->src, imm_size(c), false);
                break;
        case SrcImmByte:
+               rc = decode_imm(ctxt, &c->src, 1, true);
+               break;
        case SrcImmUByte:
-               c->src.type = OP_IMM;
-               c->src.addr.mem = c->eip;
-               c->src.bytes = 1;
-               if ((c->d & SrcMask) == SrcImmByte)
-                       c->src.val = insn_fetch(s8, 1, c->eip);
-               else
-                       c->src.val = insn_fetch(u8, 1, c->eip);
+               rc = decode_imm(ctxt, &c->src, 1, false);
                break;
        case SrcAcc:
                c->src.type = OP_REG;
@@ -2665,6 +2907,9 @@ done_prefixes:
                break;
        }
 
+       if (rc != X86EMUL_CONTINUE)
+               goto done;
+
        /*
         * Decode and fetch the second source operand: register, memory
         * or immediate.
@@ -2677,17 +2922,20 @@ done_prefixes:
                c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
                break;
        case Src2ImmByte:
-               c->src2.type = OP_IMM;
-               c->src2.addr.mem = c->eip;
-               c->src2.bytes = 1;
-               c->src2.val = insn_fetch(u8, 1, c->eip);
+               rc = decode_imm(ctxt, &c->src2, 1, true);
                break;
        case Src2One:
                c->src2.bytes = 1;
                c->src2.val = 1;
                break;
+       case Src2Imm:
+               rc = decode_imm(ctxt, &c->src2, imm_size(c), true);
+               break;
        }
 
+       if (rc != X86EMUL_CONTINUE)
+               goto done;
+
        /* Decode and fetch the destination operand: register or memory. */
        switch (c->d & DstMask) {
        case DstReg:
@@ -2737,6 +2985,28 @@ done:
        return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
 }
 
+static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
+{
+       struct decode_cache *c = &ctxt->decode;
+
+       /* The second termination condition only applies for REPE
+        * and REPNE. Test if the repeat string operation prefix is
+        * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
+        * corresponding termination condition according to:
+        *      - if REPE/REPZ and ZF = 0 then done
+        *      - if REPNE/REPNZ and ZF = 1 then done
+        */
+       if (((c->b == 0xa6) || (c->b == 0xa7) ||
+            (c->b == 0xae) || (c->b == 0xaf))
+           && (((c->rep_prefix == REPE_PREFIX) &&
+                ((ctxt->eflags & EFLG_ZF) == 0))
+               || ((c->rep_prefix == REPNE_PREFIX) &&
+                   ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
+               return true;
+
+       return false;
+}
+
 int
 x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
 {
@@ -2760,6 +3030,11 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
                goto done;
        }
 
+       if ((c->d & SrcMask) == SrcMemFAddr && c->src.type != OP_MEM) {
+               emulate_ud(ctxt);
+               goto done;
+       }
+
        /* Privileged instruction can be executed only in CPL=0 */
        if ((c->d & Priv) && ops->cpl(ctxt->vcpu)) {
                emulate_gp(ctxt, 0);
@@ -2767,31 +3042,11 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
        }
 
        if (c->rep_prefix && (c->d & String)) {
-               ctxt->restart = true;
                /* All REP prefixes have the same first termination condition */
                if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
-               string_done:
-                       ctxt->restart = false;
                        ctxt->eip = c->eip;
                        goto done;
                }
-               /* The second termination condition only applies for REPE
-                * and REPNE. Test if the repeat string operation prefix is
-                * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
-                * corresponding termination condition according to:
-                *      - if REPE/REPZ and ZF = 0 then done
-                *      - if REPNE/REPNZ and ZF = 1 then done
-                */
-               if ((c->b == 0xa6) || (c->b == 0xa7) ||
-                   (c->b == 0xae) || (c->b == 0xaf)) {
-                       if ((c->rep_prefix == REPE_PREFIX) &&
-                           ((ctxt->eflags & EFLG_ZF) == 0))
-                               goto string_done;
-                       if ((c->rep_prefix == REPNE_PREFIX) &&
-                           ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))
-                               goto string_done;
-               }
-               c->eip = ctxt->eip;
        }
 
        if ((c->src.type == OP_MEM) && !(c->d & NoAccess)) {
@@ -2844,8 +3099,6 @@ special_insn:
                break;
        case 0x07:              /* pop es */
                rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
-               if (rc != X86EMUL_CONTINUE)
-                       goto done;
                break;
        case 0x08 ... 0x0d:
              or:               /* or */
@@ -2863,8 +3116,6 @@ special_insn:
                break;
        case 0x17:              /* pop ss */
                rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
-               if (rc != X86EMUL_CONTINUE)
-                       goto done;
                break;
        case 0x18 ... 0x1d:
              sbb:              /* sbb */
@@ -2875,8 +3126,6 @@ special_insn:
                break;
        case 0x1f:              /* pop ds */
                rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
-               if (rc != X86EMUL_CONTINUE)
-                       goto done;
                break;
        case 0x20 ... 0x25:
              and:              /* and */
@@ -2903,18 +3152,12 @@ special_insn:
        case 0x58 ... 0x5f: /* pop reg */
        pop_instruction:
                rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
-               if (rc != X86EMUL_CONTINUE)
-                       goto done;
                break;
        case 0x60:      /* pusha */
                rc = emulate_pusha(ctxt, ops);
-               if (rc != X86EMUL_CONTINUE)
-                       goto done;
                break;
        case 0x61:      /* popa */
                rc = emulate_popa(ctxt, ops);
-               if (rc != X86EMUL_CONTINUE)
-                       goto done;
                break;
        case 0x63:              /* movsxd */
                if (ctxt->mode != X86EMUL_MODE_PROT64)
@@ -2923,28 +3166,12 @@ special_insn:
                break;
        case 0x6c:              /* insb */
        case 0x6d:              /* insw/insd */
-               c->dst.bytes = min(c->dst.bytes, 4u);
-               if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
-                                         c->dst.bytes)) {
-                       emulate_gp(ctxt, 0);
-                       goto done;
-               }
-               if (!pio_in_emulated(ctxt, ops, c->dst.bytes,
-                                    c->regs[VCPU_REGS_RDX], &c->dst.val))
-                       goto done; /* IO is needed, skip writeback */
-               break;
+               c->src.val = c->regs[VCPU_REGS_RDX];
+               goto do_io_in;
        case 0x6e:              /* outsb */
        case 0x6f:              /* outsw/outsd */
-               c->src.bytes = min(c->src.bytes, 4u);
-               if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
-                                         c->src.bytes)) {
-                       emulate_gp(ctxt, 0);
-                       goto done;
-               }
-               ops->pio_out_emulated(c->src.bytes, c->regs[VCPU_REGS_RDX],
-                                     &c->src.val, 1, ctxt->vcpu);
-
-               c->dst.type = OP_NONE; /* nothing to writeback */
+               c->dst.val = c->regs[VCPU_REGS_RDX];
+               goto do_io_out;
                break;
        case 0x70 ... 0x7f: /* jcc (short) */
                if (test_cc(c->b, ctxt->eflags))
@@ -2986,8 +3213,6 @@ special_insn:
                c->dst.val = c->src.orig_val;
                c->lock_prefix = 1;
                break;
-       case 0x88 ... 0x8b:     /* mov */
-               goto mov;
        case 0x8c:  /* mov r/m, sreg */
                if (c->modrm_reg > VCPU_SREG_GS) {
                        emulate_ud(ctxt);
@@ -3019,13 +3244,18 @@ special_insn:
        }
        case 0x8f:              /* pop (sole member of Grp1a) */
                rc = emulate_grp1a(ctxt, ops);
-               if (rc != X86EMUL_CONTINUE)
-                       goto done;
                break;
        case 0x90 ... 0x97: /* nop / xchg reg, rax */
                if (c->dst.addr.reg == &c->regs[VCPU_REGS_RAX])
                        break;
                goto xchg;
+       case 0x98: /* cbw/cwde/cdqe */
+               switch (c->op_bytes) {
+               case 2: c->dst.val = (s8)c->dst.val; break;
+               case 4: c->dst.val = (s16)c->dst.val; break;
+               case 8: c->dst.val = (s32)c->dst.val; break;
+               }
+               break;
        case 0x9c: /* pushf */
                c->src.val =  (unsigned long) ctxt->eflags;
                emulate_push(ctxt, ops);
@@ -3035,26 +3265,15 @@ special_insn:
                c->dst.addr.reg = &ctxt->eflags;
                c->dst.bytes = c->op_bytes;
                rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
-               if (rc != X86EMUL_CONTINUE)
-                       goto done;
                break;
-       case 0xa0 ... 0xa3:     /* mov */
-       case 0xa4 ... 0xa5:     /* movs */
-               goto mov;
        case 0xa6 ... 0xa7:     /* cmps */
                c->dst.type = OP_NONE; /* Disable writeback. */
                DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.addr.mem, c->dst.addr.mem);
                goto cmp;
        case 0xa8 ... 0xa9:     /* test ax, imm */
                goto test;
-       case 0xaa ... 0xab:     /* stos */
-       case 0xac ... 0xad:     /* lods */
-               goto mov;
        case 0xae ... 0xaf:     /* scas */
-               DPRINTF("Urk! I don't handle SCAS.\n");
-               goto cannot_emulate;
-       case 0xb0 ... 0xbf: /* mov r, imm */
-               goto mov;
+               goto cmp;
        case 0xc0 ... 0xc1:
                emulate_grp2(ctxt);
                break;
@@ -3063,14 +3282,14 @@ special_insn:
                c->dst.addr.reg = &c->eip;
                c->dst.bytes = c->op_bytes;
                goto pop_instruction;
-       case 0xc6 ... 0xc7:     /* mov (sole member of Grp11) */
-       mov:
-               c->dst.val = c->src.val;
+       case 0xc4:              /* les */
+               rc = emulate_load_segment(ctxt, ops, VCPU_SREG_ES);
+               break;
+       case 0xc5:              /* lds */
+               rc = emulate_load_segment(ctxt, ops, VCPU_SREG_DS);
                break;
        case 0xcb:              /* ret far */
                rc = emulate_ret_far(ctxt, ops);
-               if (rc != X86EMUL_CONTINUE)
-                       goto done;
                break;
        case 0xcc:              /* int3 */
                irq = 3;
@@ -3079,8 +3298,6 @@ special_insn:
                irq = c->src.val;
        do_interrupt:
                rc = emulate_int(ctxt, ops, irq);
-               if (rc != X86EMUL_CONTINUE)
-                       goto done;
                break;
        case 0xce:              /* into */
                if (ctxt->eflags & EFLG_OF) {
@@ -3090,9 +3307,6 @@ special_insn:
                break;
        case 0xcf:              /* iret */
                rc = emulate_iret(ctxt, ops);
-
-               if (rc != X86EMUL_CONTINUE)
-                       goto done;
                break;
        case 0xd0 ... 0xd1:     /* Grp2 */
                emulate_grp2(ctxt);
@@ -3101,6 +3315,16 @@ special_insn:
                c->src.val = c->regs[VCPU_REGS_RCX];
                emulate_grp2(ctxt);
                break;
+       case 0xe0 ... 0xe2:     /* loop/loopz/loopnz */
+               register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
+               if (address_mask(c, c->regs[VCPU_REGS_RCX]) != 0 &&
+                   (c->b == 0xe2 || test_cc(c->b ^ 0x5, ctxt->eflags)))
+                       jmp_rel(c, c->src.val);
+               break;
+       case 0xe3:      /* jcxz/jecxz/jrcxz */
+               if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0)
+                       jmp_rel(c, c->src.val);
+               break;
        case 0xe4:      /* inb */
        case 0xe5:      /* in */
                goto do_io_in;
@@ -3168,8 +3392,7 @@ special_insn:
                ctxt->eflags ^= EFLG_CF;
                break;
        case 0xf6 ... 0xf7:     /* Grp3 */
-               if (emulate_grp3(ctxt, ops) != X86EMUL_CONTINUE)
-                       goto cannot_emulate;
+               rc = emulate_grp3(ctxt, ops);
                break;
        case 0xf8: /* clc */
                ctxt->eflags &= ~EFLG_CF;
@@ -3202,8 +3425,6 @@ special_insn:
        case 0xfe: /* Grp4 */
        grp45:
                rc = emulate_grp45(ctxt, ops);
-               if (rc != X86EMUL_CONTINUE)
-                       goto done;
                break;
        case 0xff: /* Grp5 */
                if (c->modrm_reg == 5)
@@ -3213,6 +3434,9 @@ special_insn:
                goto cannot_emulate;
        }
 
+       if (rc != X86EMUL_CONTINUE)
+               goto done;
+
 writeback:
        rc = writeback(ctxt, ops);
        if (rc != X86EMUL_CONTINUE)
@@ -3233,25 +3457,32 @@ writeback:
                                &c->dst);
 
        if (c->rep_prefix && (c->d & String)) {
-               struct read_cache *rc = &ctxt->decode.io_read;
+               struct read_cache *r = &ctxt->decode.io_read;
                register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
-               /*
-                * Re-enter guest when pio read ahead buffer is empty or,
-                * if it is not used, after each 1024 iteration.
-                */
-               if ((rc->end == 0 && !(c->regs[VCPU_REGS_RCX] & 0x3ff)) ||
-                   (rc->end != 0 && rc->end == rc->pos))
-                       ctxt->restart = false;
+
+               if (!string_insn_completed(ctxt)) {
+                       /*
+                        * Re-enter guest when pio read ahead buffer is empty
+                        * or, if it is not used, after each 1024 iteration.
+                        */
+                       if ((r->end != 0 || c->regs[VCPU_REGS_RCX] & 0x3ff) &&
+                           (r->end == 0 || r->end != r->pos)) {
+                               /*
+                                * Reset read cache. Usually happens before
+                                * decode, but since instruction is restarted
+                                * we have to do it here.
+                                */
+                               ctxt->decode.mem_read.end = 0;
+                               return EMULATION_RESTART;
+                       }
+                       goto done; /* skip rip writeback */
+               }
        }
-       /*
-        * reset read cache here in case string instruction is restared
-        * without decoding
-        */
-       ctxt->decode.mem_read.end = 0;
+
        ctxt->eip = c->eip;
 
 done:
-       return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
+       return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
 
 twobyte_insn:
        switch (c->b) {
@@ -3287,8 +3518,6 @@ twobyte_insn:
                                switch (c->modrm_rm) {
                                case 1:
                                        rc = kvm_fix_hypercall(ctxt->vcpu);
-                                       if (rc != X86EMUL_CONTINUE)
-                                               goto done;
                                        break;
                                default:
                                        goto cannot_emulate;
@@ -3327,10 +3556,6 @@ twobyte_insn:
                break;
        case 0x05:              /* syscall */
                rc = emulate_syscall(ctxt, ops);
-               if (rc != X86EMUL_CONTINUE)
-                       goto done;
-               else
-                       goto writeback;
                break;
        case 0x06:
                emulate_clts(ctxt->vcpu);
@@ -3407,17 +3632,9 @@ twobyte_insn:
                break;
        case 0x34:              /* sysenter */
                rc = emulate_sysenter(ctxt, ops);
-               if (rc != X86EMUL_CONTINUE)
-                       goto done;
-               else
-                       goto writeback;
                break;
        case 0x35:              /* sysexit */
                rc = emulate_sysexit(ctxt, ops);
-               if (rc != X86EMUL_CONTINUE)
-                       goto done;
-               else
-                       goto writeback;
                break;
        case 0x40 ... 0x4f:     /* cmov */
                c->dst.val = c->dst.orig_val = c->src.val;
@@ -3436,8 +3653,6 @@ twobyte_insn:
                break;
        case 0xa1:       /* pop fs */
                rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
-               if (rc != X86EMUL_CONTINUE)
-                       goto done;
                break;
        case 0xa3:
              bt:               /* bt */
@@ -3455,8 +3670,6 @@ twobyte_insn:
                break;
        case 0xa9:      /* pop gs */
                rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
-               if (rc != X86EMUL_CONTINUE)
-                       goto done;
                break;
        case 0xab:
              bts:              /* bts */
@@ -3485,10 +3698,19 @@ twobyte_insn:
                        c->dst.addr.reg = (unsigned long *)&c->regs[VCPU_REGS_RAX];
                }
                break;
+       case 0xb2:              /* lss */
+               rc = emulate_load_segment(ctxt, ops, VCPU_SREG_SS);
+               break;
        case 0xb3:
              btr:              /* btr */
                emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
                break;
+       case 0xb4:              /* lfs */
+               rc = emulate_load_segment(ctxt, ops, VCPU_SREG_FS);
+               break;
+       case 0xb5:              /* lgs */
+               rc = emulate_load_segment(ctxt, ops, VCPU_SREG_GS);
+               break;
        case 0xb6 ... 0xb7:     /* movzx */
                c->dst.bytes = c->op_bytes;
                c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
@@ -3552,12 +3774,14 @@ twobyte_insn:
                break;
        case 0xc7:              /* Grp9 (cmpxchg8b) */
                rc = emulate_grp9(ctxt, ops);
-               if (rc != X86EMUL_CONTINUE)
-                       goto done;
                break;
        default:
                goto cannot_emulate;
        }
+
+       if (rc != X86EMUL_CONTINUE)
+               goto done;
+
        goto writeback;
 
 cannot_emulate: