arch/x86/kvm/svm.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * AMD SVM support
   5  *
   6  * Copyright (C) 2006 Qumranet, Inc.
   7  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
   8  *
   9  * Authors:
  10  *   Yaniv Kamay  <yaniv@qumranet.com>
  11  *   Avi Kivity   <avi@qumranet.com>
  12  *
  13  * This work is licensed under the terms of the GNU GPL, version 2.  See
  14  * the COPYING file in the top-level directory.
  15  *
  16  */
  17 #include <linux/kvm_host.h>
  18
  19 #include "irq.h"
  20 #include "mmu.h"
  21 #include "kvm_cache_regs.h"
  22 #include "x86.h"
  23
  24 #include <linux/module.h>
  25 #include <linux/kernel.h>
  26 #include <linux/vmalloc.h>
  27 #include <linux/highmem.h>
  28 #include <linux/sched.h>
  29 #include <linux/ftrace_event.h>
  30 #include <linux/slab.h>
  31
  32 #include <asm/tlbflush.h>
  33 #include <asm/desc.h>
  34 #include <asm/kvm_para.h>
  35
  36 #include <asm/virtext.h>
  37 #include "trace.h"
  38
  39 #define __ex(x) __kvm_handle_fault_on_reboot(x)
  40
  41 MODULE_AUTHOR("Qumranet");
  42 MODULE_LICENSE("GPL");
  43
  44 #define IOPM_ALLOC_ORDER 2
  45 #define MSRPM_ALLOC_ORDER 1
  46
  47 #define SEG_TYPE_LDT 2
  48 #define SEG_TYPE_BUSY_TSS16 3
  49
  50 #define SVM_FEATURE_NPT            (1 <<  0)
  51 #define SVM_FEATURE_LBRV           (1 <<  1)
  52 #define SVM_FEATURE_SVML           (1 <<  2)
  53 #define SVM_FEATURE_NRIP           (1 <<  3)
  54 #define SVM_FEATURE_PAUSE_FILTER   (1 << 10)
  55
  56 #define NESTED_EXIT_HOST        0       /* Exit handled on host level */
  57 #define NESTED_EXIT_DONE        1       /* Exit caused nested vmexit  */
  58 #define NESTED_EXIT_CONTINUE    2       /* Further checks needed      */
  59
  60 #define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
  61
  62 static bool erratum_383_found __read_mostly;
  63
  64 static const u32 host_save_user_msrs[] = {
  65 #ifdef CONFIG_X86_64
  66         MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE,
  67         MSR_FS_BASE,
  68 #endif
  69         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
  70 };
  71
  72 #define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)
  73
  74 struct kvm_vcpu;
  75
  76 struct nested_state {
  77         struct vmcb *hsave;
  78         u64 hsave_msr;
  79         u64 vm_cr_msr;
  80         u64 vmcb;
  81
  82         /* These are the merged vectors */
  83         u32 *msrpm;
  84
  85         /* gpa pointers to the real vectors */
  86         u64 vmcb_msrpm;
  87         u64 vmcb_iopm;
  88
  89         /* A VMEXIT is required but not yet emulated */
  90         bool exit_required;
  91
  92         /*
  93          * If we vmexit during an instruction emulation we need this to restore
  94          * the l1 guest rip after the emulation
  95          */
  96         unsigned long vmexit_rip;
  97         unsigned long vmexit_rsp;
  98         unsigned long vmexit_rax;
  99
 100         /* cache for intercepts of the guest */
 101         u32 intercept_cr;
 102         u16 intercept_dr_read;
 103         u16 intercept_dr_write;
 104         u32 intercept_exceptions;
 105         u64 intercept;
 106
 107         /* Nested Paging related state */
 108         u64 nested_cr3;
 109 };
 110
 111 #define MSRPM_OFFSETS   16
 112 static u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
 113
 114 struct vcpu_svm {
 115         struct kvm_vcpu vcpu;
 116         struct vmcb *vmcb;
 117         unsigned long vmcb_pa;
 118         struct svm_cpu_data *svm_data;
 119         uint64_t asid_generation;
 120         uint64_t sysenter_esp;
 121         uint64_t sysenter_eip;
 122
 123         u64 next_rip;
 124
 125         u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
 126         struct {
 127                 u16 fs;
 128                 u16 gs;
 129                 u16 ldt;
 130                 u64 gs_base;
 131         } host;
 132
 133         u32 *msrpm;
 134
 135         struct nested_state nested;
 136
 137         bool nmi_singlestep;
 138
 139         unsigned int3_injected;
 140         unsigned long int3_rip;
 141         u32 apf_reason;
 142 };
 143
 144 #define MSR_INVALID                     0xffffffffU
 145
 146 static struct svm_direct_access_msrs {
 147         u32 index;   /* Index of the MSR */
 148         bool always; /* True if intercept is always on */
 149 } direct_access_msrs[] = {
 150         { .index = MSR_STAR,                            .always = true  },
 151         { .index = MSR_IA32_SYSENTER_CS,                .always = true  },
 152 #ifdef CONFIG_X86_64
 153         { .index = MSR_GS_BASE,                         .always = true  },
 154         { .index = MSR_FS_BASE,                         .always = true  },
 155         { .index = MSR_KERNEL_GS_BASE,                  .always = true  },
 156         { .index = MSR_LSTAR,                           .always = true  },
 157         { .index = MSR_CSTAR,                           .always = true  },
 158         { .index = MSR_SYSCALL_MASK,                    .always = true  },
 159 #endif
 160         { .index = MSR_IA32_LASTBRANCHFROMIP,           .always = false },
 161         { .index = MSR_IA32_LASTBRANCHTOIP,             .always = false },
 162         { .index = MSR_IA32_LASTINTFROMIP,              .always = false },
 163         { .index = MSR_IA32_LASTINTTOIP,                .always = false },
 164         { .index = MSR_INVALID,                         .always = false },
 165 };
 166
 167 /* enable NPT for AMD64 and X86 with PAE */
 168 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
 169 static bool npt_enabled = true;
 170 #else
 171 static bool npt_enabled;
 172 #endif
 173 static int npt = 1;
 174
 175 module_param(npt, int, S_IRUGO);
 176
 177 static int nested = 1;
 178 module_param(nested, int, S_IRUGO);
 179
 180 static void svm_flush_tlb(struct kvm_vcpu *vcpu);
 181 static void svm_complete_interrupts(struct vcpu_svm *svm);
 182
 183 static int nested_svm_exit_handled(struct vcpu_svm *svm);
 184 static int nested_svm_intercept(struct vcpu_svm *svm);
 185 static int nested_svm_vmexit(struct vcpu_svm *svm);
 186 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
 187                                       bool has_error_code, u32 error_code);
 188
 189 static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
 190 {
 191         return container_of(vcpu, struct vcpu_svm, vcpu);
 192 }
 193
 194 static void recalc_intercepts(struct vcpu_svm *svm)
 195 {
 196         struct vmcb_control_area *c, *h;
 197         struct nested_state *g;
 198
 199         if (!is_guest_mode(&svm->vcpu))
 200                 return;
 201
 202         c = &svm->vmcb->control;
 203         h = &svm->nested.hsave->control;
 204         g = &svm->nested;
 205
 206         c->intercept_cr = h->intercept_cr | g->intercept_cr;
 207         c->intercept_dr_read = h->intercept_dr_read | g->intercept_dr_read;
 208         c->intercept_dr_write = h->intercept_dr_write | g->intercept_dr_write;
 209         c->intercept_exceptions = h->intercept_exceptions | g->intercept_exceptions;
 210         c->intercept = h->intercept | g->intercept;
 211 }
 212
 213 static inline struct vmcb *get_host_vmcb(struct vcpu_svm *svm)
 214 {
 215         if (is_guest_mode(&svm->vcpu))
 216                 return svm->nested.hsave;
 217         else
 218                 return svm->vmcb;
 219 }
 220
 221 static inline void set_cr_intercept(struct vcpu_svm *svm, int bit)
 222 {
 223         struct vmcb *vmcb = get_host_vmcb(svm);
 224
 225         vmcb->control.intercept_cr |= (1U << bit);
 226
 227         recalc_intercepts(svm);
 228 }
 229
 230 static inline void clr_cr_intercept(struct vcpu_svm *svm, int bit)
 231 {
 232         struct vmcb *vmcb = get_host_vmcb(svm);
 233
 234         vmcb->control.intercept_cr &= ~(1U << bit);
 235
 236         recalc_intercepts(svm);
 237 }
 238
 239 static inline bool is_cr_intercept(struct vcpu_svm *svm, int bit)
 240 {
 241         struct vmcb *vmcb = get_host_vmcb(svm);
 242
 243         return vmcb->control.intercept_cr & (1U << bit);
 244 }
 245
 246 static inline void enable_gif(struct vcpu_svm *svm)
 247 {
 248         svm->vcpu.arch.hflags |= HF_GIF_MASK;
 249 }
 250
 251 static inline void disable_gif(struct vcpu_svm *svm)
 252 {
 253         svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
 254 }
 255
 256 static inline bool gif_set(struct vcpu_svm *svm)
 257 {
 258         return !!(svm->vcpu.arch.hflags & HF_GIF_MASK);
 259 }
 260
 261 static unsigned long iopm_base;
 262
 263 struct kvm_ldttss_desc {
 264         u16 limit0;
 265         u16 base0;
 266         unsigned base1:8, type:5, dpl:2, p:1;
 267         unsigned limit1:4, zero0:3, g:1, base2:8;
 268         u32 base3;
 269         u32 zero1;
 270 } __attribute__((packed));
 271
 272 struct svm_cpu_data {
 273         int cpu;
 274
 275         u64 asid_generation;
 276         u32 max_asid;
 277         u32 next_asid;
 278         struct kvm_ldttss_desc *tss_desc;
 279
 280         struct page *save_area;
 281 };
 282
 283 static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
 284 static uint32_t svm_features;
 285
 286 struct svm_init_data {
 287         int cpu;
 288         int r;
 289 };
 290
 291 static u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
 292
 293 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
 294 #define MSRS_RANGE_SIZE 2048
 295 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
 296
 297 static u32 svm_msrpm_offset(u32 msr)
 298 {
 299         u32 offset;
 300         int i;
 301
 302         for (i = 0; i < NUM_MSR_MAPS; i++) {
 303                 if (msr < msrpm_ranges[i] ||
 304                     msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
 305                         continue;
 306
 307                 offset  = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
 308                 offset += (i * MSRS_RANGE_SIZE);       /* add range offset */
 309
 310                 /* Now we have the u8 offset - but need the u32 offset */
 311                 return offset / 4;
 312         }
 313
 314         /* MSR not in any range */
 315         return MSR_INVALID;
 316 }
 317
 318 #define MAX_INST_SIZE 15
 319
 320 static inline void clgi(void)
 321 {
 322         asm volatile (__ex(SVM_CLGI));
 323 }
 324
 325 static inline void stgi(void)
 326 {
 327         asm volatile (__ex(SVM_STGI));
 328 }
 329
 330 static inline void invlpga(unsigned long addr, u32 asid)
 331 {
 332         asm volatile (__ex(SVM_INVLPGA) : : "a"(addr), "c"(asid));
 333 }
 334
 335 static inline void force_new_asid(struct kvm_vcpu *vcpu)
 336 {
 337         to_svm(vcpu)->asid_generation--;
 338 }
 339
 340 static inline void flush_guest_tlb(struct kvm_vcpu *vcpu)
 341 {
 342         force_new_asid(vcpu);
 343 }
 344
 345 static int get_npt_level(void)
 346 {
 347 #ifdef CONFIG_X86_64
 348         return PT64_ROOT_LEVEL;
 349 #else
 350         return PT32E_ROOT_LEVEL;
 351 #endif
 352 }
 353
 354 static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
 355 {
 356         vcpu->arch.efer = efer;
 357         if (!npt_enabled && !(efer & EFER_LMA))
 358                 efer &= ~EFER_LME;
 359
 360         to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME;
 361 }
 362
 363 static int is_external_interrupt(u32 info)
 364 {
 365         info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
 366         return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
 367 }
 368
 369 static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
 370 {
 371         struct vcpu_svm *svm = to_svm(vcpu);
 372         u32 ret = 0;
 373
 374         if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
 375                 ret |= KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
 376         return ret & mask;
 377 }
 378
 379 static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
 380 {
 381         struct vcpu_svm *svm = to_svm(vcpu);
 382
 383         if (mask == 0)
 384                 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
 385         else
 386                 svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
 387
 388 }
 389
 390 static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
 391 {
 392         struct vcpu_svm *svm = to_svm(vcpu);
 393
 394         if (svm->vmcb->control.next_rip != 0)
 395                 svm->next_rip = svm->vmcb->control.next_rip;
 396
 397         if (!svm->next_rip) {
 398                 if (emulate_instruction(vcpu, 0, 0, EMULTYPE_SKIP) !=
 399                                 EMULATE_DONE)
 400                         printk(KERN_DEBUG "%s: NOP\n", __func__);
 401                 return;
 402         }
 403         if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)
 404                 printk(KERN_ERR "%s: ip 0x%lx next 0x%llx\n",
 405                        __func__, kvm_rip_read(vcpu), svm->next_rip);
 406
 407         kvm_rip_write(vcpu, svm->next_rip);
 408         svm_set_interrupt_shadow(vcpu, 0);
 409 }
 410
 411 static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
 412                                 bool has_error_code, u32 error_code,
 413                                 bool reinject)
 414 {
 415         struct vcpu_svm *svm = to_svm(vcpu);
 416
 417         /*
 418          * If we are within a nested VM we'd better #VMEXIT and let the guest
 419          * handle the exception
 420          */
 421         if (!reinject &&
 422             nested_svm_check_exception(svm, nr, has_error_code, error_code))
 423                 return;
 424
 425         if (nr == BP_VECTOR && !static_cpu_has(X86_FEATURE_NRIPS)) {
 426                 unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu);
 427
 428                 /*
 429                  * For guest debugging where we have to reinject #BP if some
 430                  * INT3 is guest-owned:
 431                  * Emulate nRIP by moving RIP forward. Will fail if injection
 432                  * raises a fault that is not intercepted. Still better than
 433                  * failing in all cases.
 434                  */
 435                 skip_emulated_instruction(&svm->vcpu);
 436                 rip = kvm_rip_read(&svm->vcpu);
 437                 svm->int3_rip = rip + svm->vmcb->save.cs.base;
 438                 svm->int3_injected = rip - old_rip;
 439         }
 440
 441         svm->vmcb->control.event_inj = nr
 442                 | SVM_EVTINJ_VALID
 443                 | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
 444                 | SVM_EVTINJ_TYPE_EXEPT;
 445         svm->vmcb->control.event_inj_err = error_code;
 446 }
 447
 448 static void svm_init_erratum_383(void)
 449 {
 450         u32 low, high;
 451         int err;
 452         u64 val;
 453
 454         if (!cpu_has_amd_erratum(amd_erratum_383))
 455                 return;
 456
 457         /* Use _safe variants to not break nested virtualization */
 458         val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
 459         if (err)
 460                 return;
 461
 462         val |= (1ULL << 47);
 463
 464         low  = lower_32_bits(val);
 465         high = upper_32_bits(val);
 466
 467         native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);
 468
 469         erratum_383_found = true;
 470 }
 471
 472 static int has_svm(void)
 473 {
 474         const char *msg;
 475
 476         if (!cpu_has_svm(&msg)) {
 477                 printk(KERN_INFO "has_svm: %s\n", msg);
 478                 return 0;
 479         }
 480
 481         return 1;
 482 }
 483
 484 static void svm_hardware_disable(void *garbage)
 485 {
 486         cpu_svm_disable();
 487 }
 488
 489 static int svm_hardware_enable(void *garbage)
 490 {
 491
 492         struct svm_cpu_data *sd;
 493         uint64_t efer;
 494         struct desc_ptr gdt_descr;
 495         struct desc_struct *gdt;
 496         int me = raw_smp_processor_id();
 497
 498         rdmsrl(MSR_EFER, efer);
 499         if (efer & EFER_SVME)
 500                 return -EBUSY;
 501
 502         if (!has_svm()) {
 503                 printk(KERN_ERR "svm_hardware_enable: err EOPNOTSUPP on %d\n",
 504                        me);
 505                 return -EINVAL;
 506         }
 507         sd = per_cpu(svm_data, me);
 508
 509         if (!sd) {
 510                 printk(KERN_ERR "svm_hardware_enable: svm_data is NULL on %d\n",
 511                        me);
 512                 return -EINVAL;
 513         }
 514
 515         sd->asid_generation = 1;
 516         sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
 517         sd->next_asid = sd->max_asid + 1;
 518
 519         native_store_gdt(&gdt_descr);
 520         gdt = (struct desc_struct *)gdt_descr.address;
 521         sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
 522
 523         wrmsrl(MSR_EFER, efer | EFER_SVME);
 524
 525         wrmsrl(MSR_VM_HSAVE_PA, page_to_pfn(sd->save_area) << PAGE_SHIFT);
 526
 527         svm_init_erratum_383();
 528
 529         return 0;
 530 }
 531
 532 static void svm_cpu_uninit(int cpu)
 533 {
 534         struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id());
 535
 536         if (!sd)
 537                 return;
 538
 539         per_cpu(svm_data, raw_smp_processor_id()) = NULL;
 540         __free_page(sd->save_area);
 541         kfree(sd);
 542 }
 543
 544 static int svm_cpu_init(int cpu)
 545 {
 546         struct svm_cpu_data *sd;
 547         int r;
 548
 549         sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
 550         if (!sd)
 551                 return -ENOMEM;
 552         sd->cpu = cpu;
 553         sd->save_area = alloc_page(GFP_KERNEL);
 554         r = -ENOMEM;
 555         if (!sd->save_area)
 556                 goto err_1;
 557
 558         per_cpu(svm_data, cpu) = sd;
 559
 560         return 0;
 561
 562 err_1:
 563         kfree(sd);
 564         return r;
 565
 566 }
 567
 568 static bool valid_msr_intercept(u32 index)
 569 {
 570         int i;
 571
 572         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
 573                 if (direct_access_msrs[i].index == index)
 574                         return true;
 575
 576         return false;
 577 }
 578
 579 static void set_msr_interception(u32 *msrpm, unsigned msr,
 580                                  int read, int write)
 581 {
 582         u8 bit_read, bit_write;
 583         unsigned long tmp;
 584         u32 offset;
 585
 586         /*
 587          * If this warning triggers extend the direct_access_msrs list at the
 588          * beginning of the file
 589          */
 590         WARN_ON(!valid_msr_intercept(msr));
 591
 592         offset    = svm_msrpm_offset(msr);
 593         bit_read  = 2 * (msr & 0x0f);
 594         bit_write = 2 * (msr & 0x0f) + 1;
 595         tmp       = msrpm[offset];
 596
 597         BUG_ON(offset == MSR_INVALID);
 598
 599         read  ? clear_bit(bit_read,  &tmp) : set_bit(bit_read,  &tmp);
 600         write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);
 601
 602         msrpm[offset] = tmp;
 603 }
 604
 605 static void svm_vcpu_init_msrpm(u32 *msrpm)
 606 {
 607         int i;
 608
 609         memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));
 610
 611         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
 612                 if (!direct_access_msrs[i].always)
 613                         continue;
 614
 615                 set_msr_interception(msrpm, direct_access_msrs[i].index, 1, 1);
 616         }
 617 }
 618
 619 static void add_msr_offset(u32 offset)
 620 {
 621         int i;
 622
 623         for (i = 0; i < MSRPM_OFFSETS; ++i) {
 624
 625                 /* Offset already in list? */
 626                 if (msrpm_offsets[i] == offset)
 627                         return;
 628
 629                 /* Slot used by another offset? */
 630                 if (msrpm_offsets[i] != MSR_INVALID)
 631                         continue;
 632
 633                 /* Add offset to list */
 634                 msrpm_offsets[i] = offset;
 635
 636                 return;
 637         }
 638
 639         /*
 640          * If this BUG triggers the msrpm_offsets table has an overflow. Just
 641          * increase MSRPM_OFFSETS in this case.
 642          */
 643         BUG();
 644 }
 645
 646 static void init_msrpm_offsets(void)
 647 {
 648         int i;
 649
 650         memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));
 651
 652         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
 653                 u32 offset;
 654
 655                 offset = svm_msrpm_offset(direct_access_msrs[i].index);
 656                 BUG_ON(offset == MSR_INVALID);
 657
 658                 add_msr_offset(offset);
 659         }
 660 }
 661
 662 static void svm_enable_lbrv(struct vcpu_svm *svm)
 663 {
 664         u32 *msrpm = svm->msrpm;
 665
 666         svm->vmcb->control.lbr_ctl = 1;
 667         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
 668         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
 669         set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
 670         set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
 671 }
 672
 673 static void svm_disable_lbrv(struct vcpu_svm *svm)
 674 {
 675         u32 *msrpm = svm->msrpm;
 676
 677         svm->vmcb->control.lbr_ctl = 0;
 678         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
 679         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
 680         set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
 681         set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
 682 }
 683
 684 static __init int svm_hardware_setup(void)
 685 {
 686         int cpu;
 687         struct page *iopm_pages;
 688         void *iopm_va;
 689         int r;
 690
 691         iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);
 692
 693         if (!iopm_pages)
 694                 return -ENOMEM;
 695
 696         iopm_va = page_address(iopm_pages);
 697         memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
 698         iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
 699
 700         init_msrpm_offsets();
 701
 702         if (boot_cpu_has(X86_FEATURE_NX))
 703                 kvm_enable_efer_bits(EFER_NX);
 704
 705         if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
 706                 kvm_enable_efer_bits(EFER_FFXSR);
 707
 708         if (nested) {
 709                 printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
 710                 kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
 711         }
 712
 713         for_each_possible_cpu(cpu) {
 714                 r = svm_cpu_init(cpu);
 715                 if (r)
 716                         goto err;
 717         }
 718
 719         svm_features = cpuid_edx(SVM_CPUID_FUNC);
 720
 721         if (!boot_cpu_has(X86_FEATURE_NPT))
 722                 npt_enabled = false;
 723
 724         if (npt_enabled && !npt) {
 725                 printk(KERN_INFO "kvm: Nested Paging disabled\n");
 726                 npt_enabled = false;
 727         }
 728
 729         if (npt_enabled) {
 730                 printk(KERN_INFO "kvm: Nested Paging enabled\n");
 731                 kvm_enable_tdp();
 732         } else
 733                 kvm_disable_tdp();
 734
 735         return 0;
 736
 737 err:
 738         __free_pages(iopm_pages, IOPM_ALLOC_ORDER);
 739         iopm_base = 0;
 740         return r;
 741 }
 742
 743 static __exit void svm_hardware_unsetup(void)
 744 {
 745         int cpu;
 746
 747         for_each_possible_cpu(cpu)
 748                 svm_cpu_uninit(cpu);
 749
 750         __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
 751         iopm_base = 0;
 752 }
 753
 754 static void init_seg(struct vmcb_seg *seg)
 755 {
 756         seg->selector = 0;
 757         seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
 758                       SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
 759         seg->limit = 0xffff;
 760         seg->base = 0;
 761 }
 762
 763 static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
 764 {
 765         seg->selector = 0;
 766         seg->attrib = SVM_SELECTOR_P_MASK | type;
 767         seg->limit = 0xffff;
 768         seg->base = 0;
 769 }
 770
 771 static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
 772 {
 773         struct vcpu_svm *svm = to_svm(vcpu);
 774         u64 g_tsc_offset = 0;
 775
 776         if (is_guest_mode(vcpu)) {
 777                 g_tsc_offset = svm->vmcb->control.tsc_offset -
 778                                svm->nested.hsave->control.tsc_offset;
 779                 svm->nested.hsave->control.tsc_offset = offset;
 780         }
 781
 782         svm->vmcb->control.tsc_offset = offset + g_tsc_offset;
 783 }
 784
 785 static void svm_adjust_tsc_offset(struct kvm_vcpu *vcpu, s64 adjustment)
 786 {
 787         struct vcpu_svm *svm = to_svm(vcpu);
 788
 789         svm->vmcb->control.tsc_offset += adjustment;
 790         if (is_guest_mode(vcpu))
 791                 svm->nested.hsave->control.tsc_offset += adjustment;
 792 }
 793
 794 static void init_vmcb(struct vcpu_svm *svm)
 795 {
 796         struct vmcb_control_area *control = &svm->vmcb->control;
 797         struct vmcb_save_area *save = &svm->vmcb->save;
 798
 799         svm->vcpu.fpu_active = 1;
 800         svm->vcpu.arch.hflags = 0;
 801
 802         set_cr_intercept(svm, INTERCEPT_CR0_READ);
 803         set_cr_intercept(svm, INTERCEPT_CR3_READ);
 804         set_cr_intercept(svm, INTERCEPT_CR4_READ);
 805         set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
 806         set_cr_intercept(svm, INTERCEPT_CR3_WRITE);
 807         set_cr_intercept(svm, INTERCEPT_CR4_WRITE);
 808         set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
 809
 810         control->intercept_dr_read =    INTERCEPT_DR0_MASK |
 811                                         INTERCEPT_DR1_MASK |
 812                                         INTERCEPT_DR2_MASK |
 813                                         INTERCEPT_DR3_MASK |
 814                                         INTERCEPT_DR4_MASK |
 815                                         INTERCEPT_DR5_MASK |
 816                                         INTERCEPT_DR6_MASK |
 817                                         INTERCEPT_DR7_MASK;
 818
 819         control->intercept_dr_write =   INTERCEPT_DR0_MASK |
 820                                         INTERCEPT_DR1_MASK |
 821                                         INTERCEPT_DR2_MASK |
 822                                         INTERCEPT_DR3_MASK |
 823                                         INTERCEPT_DR4_MASK |
 824                                         INTERCEPT_DR5_MASK |
 825                                         INTERCEPT_DR6_MASK |
 826                                         INTERCEPT_DR7_MASK;
 827
 828         control->intercept_exceptions = (1 << PF_VECTOR) |
 829                                         (1 << UD_VECTOR) |
 830                                         (1 << MC_VECTOR);
 831
 832
 833         control->intercept =    (1ULL << INTERCEPT_INTR) |
 834                                 (1ULL << INTERCEPT_NMI) |
 835                                 (1ULL << INTERCEPT_SMI) |
 836                                 (1ULL << INTERCEPT_SELECTIVE_CR0) |
 837                                 (1ULL << INTERCEPT_CPUID) |
 838                                 (1ULL << INTERCEPT_INVD) |
 839                                 (1ULL << INTERCEPT_HLT) |
 840                                 (1ULL << INTERCEPT_INVLPG) |
 841                                 (1ULL << INTERCEPT_INVLPGA) |
 842                                 (1ULL << INTERCEPT_IOIO_PROT) |
 843                                 (1ULL << INTERCEPT_MSR_PROT) |
 844                                 (1ULL << INTERCEPT_TASK_SWITCH) |
 845                                 (1ULL << INTERCEPT_SHUTDOWN) |
 846                                 (1ULL << INTERCEPT_VMRUN) |
 847                                 (1ULL << INTERCEPT_VMMCALL) |
 848                                 (1ULL << INTERCEPT_VMLOAD) |
 849                                 (1ULL << INTERCEPT_VMSAVE) |
 850                                 (1ULL << INTERCEPT_STGI) |
 851                                 (1ULL << INTERCEPT_CLGI) |
 852                                 (1ULL << INTERCEPT_SKINIT) |
 853                                 (1ULL << INTERCEPT_WBINVD) |
 854                                 (1ULL << INTERCEPT_MONITOR) |
 855                                 (1ULL << INTERCEPT_MWAIT);
 856
 857         control->iopm_base_pa = iopm_base;
 858         control->msrpm_base_pa = __pa(svm->msrpm);
 859         control->int_ctl = V_INTR_MASKING_MASK;
 860
 861         init_seg(&save->es);
 862         init_seg(&save->ss);
 863         init_seg(&save->ds);
 864         init_seg(&save->fs);
 865         init_seg(&save->gs);
 866
 867         save->cs.selector = 0xf000;
 868         /* Executable/Readable Code Segment */
 869         save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
 870                 SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
 871         save->cs.limit = 0xffff;
 872         /*
 873          * cs.base should really be 0xffff0000, but vmx can't handle that, so
 874          * be consistent with it.
 875          *
 876          * Replace when we have real mode working for vmx.
 877          */
 878         save->cs.base = 0xf0000;
 879
 880         save->gdtr.limit = 0xffff;
 881         save->idtr.limit = 0xffff;
 882
 883         init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
 884         init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
 885
 886         svm_set_efer(&svm->vcpu, 0);
 887         save->dr6 = 0xffff0ff0;
 888         save->dr7 = 0x400;
 889         save->rflags = 2;
 890         save->rip = 0x0000fff0;
 891         svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
 892
 893         /*
 894          * This is the guest-visible cr0 value.
 895          * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0.
 896          */
 897         svm->vcpu.arch.cr0 = 0;
 898         (void)kvm_set_cr0(&svm->vcpu, X86_CR0_NW | X86_CR0_CD | X86_CR0_ET);
 899
 900         save->cr4 = X86_CR4_PAE;
 901         /* rdx = ?? */
 902
 903         if (npt_enabled) {
 904                 /* Setup VMCB for Nested Paging */
 905                 control->nested_ctl = 1;
 906                 control->intercept &= ~((1ULL << INTERCEPT_TASK_SWITCH) |
 907                                         (1ULL << INTERCEPT_INVLPG));
 908                 control->intercept_exceptions &= ~(1 << PF_VECTOR);
 909                 clr_cr_intercept(svm, INTERCEPT_CR3_READ);
 910                 clr_cr_intercept(svm, INTERCEPT_CR3_WRITE);
 911                 save->g_pat = 0x0007040600070406ULL;
 912                 save->cr3 = 0;
 913                 save->cr4 = 0;
 914         }
 915         force_new_asid(&svm->vcpu);
 916
 917         svm->nested.vmcb = 0;
 918         svm->vcpu.arch.hflags = 0;
 919
 920         if (boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
 921                 control->pause_filter_count = 3000;
 922                 control->intercept |= (1ULL << INTERCEPT_PAUSE);
 923         }
 924
 925         enable_gif(svm);
 926 }
 927
 928 static int svm_vcpu_reset(struct kvm_vcpu *vcpu)
 929 {
 930         struct vcpu_svm *svm = to_svm(vcpu);
 931
 932         init_vmcb(svm);
 933
 934         if (!kvm_vcpu_is_bsp(vcpu)) {
 935                 kvm_rip_write(vcpu, 0);
 936                 svm->vmcb->save.cs.base = svm->vcpu.arch.sipi_vector << 12;
 937                 svm->vmcb->save.cs.selector = svm->vcpu.arch.sipi_vector << 8;
 938         }
 939         vcpu->arch.regs_avail = ~0;
 940         vcpu->arch.regs_dirty = ~0;
 941
 942         return 0;
 943 }
 944
 945 static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
 946 {
 947         struct vcpu_svm *svm;
 948         struct page *page;
 949         struct page *msrpm_pages;
 950         struct page *hsave_page;
 951         struct page *nested_msrpm_pages;
 952         int err;
 953
 954         svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
 955         if (!svm) {
 956                 err = -ENOMEM;
 957                 goto out;
 958         }
 959
 960         err = kvm_vcpu_init(&svm->vcpu, kvm, id);
 961         if (err)
 962                 goto free_svm;
 963
 964         err = -ENOMEM;
 965         page = alloc_page(GFP_KERNEL);
 966         if (!page)
 967                 goto uninit;
 968
 969         msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
 970         if (!msrpm_pages)
 971                 goto free_page1;
 972
 973         nested_msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
 974         if (!nested_msrpm_pages)
 975                 goto free_page2;
 976
 977         hsave_page = alloc_page(GFP_KERNEL);
 978         if (!hsave_page)
 979                 goto free_page3;
 980
 981         svm->nested.hsave = page_address(hsave_page);
 982
 983         svm->msrpm = page_address(msrpm_pages);
 984         svm_vcpu_init_msrpm(svm->msrpm);
 985
 986         svm->nested.msrpm = page_address(nested_msrpm_pages);
 987         svm_vcpu_init_msrpm(svm->nested.msrpm);
 988
 989         svm->vmcb = page_address(page);
 990         clear_page(svm->vmcb);
 991         svm->vmcb_pa = page_to_pfn(page) << PAGE_SHIFT;
 992         svm->asid_generation = 0;
 993         init_vmcb(svm);
 994         kvm_write_tsc(&svm->vcpu, 0);
 995
 996         err = fx_init(&svm->vcpu);
 997         if (err)
 998                 goto free_page4;
 999
1000         svm->vcpu.arch.apic_base = 0xfee00000 | MSR_IA32_APICBASE_ENABLE;
1001         if (kvm_vcpu_is_bsp(&svm->vcpu))
1002                 svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
1003
1004         return &svm->vcpu;
1005
1006 free_page4:
1007         __free_page(hsave_page);
1008 free_page3:
1009         __free_pages(nested_msrpm_pages, MSRPM_ALLOC_ORDER);
1010 free_page2:
1011         __free_pages(msrpm_pages, MSRPM_ALLOC_ORDER);
1012 free_page1:
1013         __free_page(page);
1014 uninit:
1015         kvm_vcpu_uninit(&svm->vcpu);
1016 free_svm:
1017         kmem_cache_free(kvm_vcpu_cache, svm);
1018 out:
1019         return ERR_PTR(err);
1020 }
1021
1022 static void svm_free_vcpu(struct kvm_vcpu *vcpu)
1023 {
1024         struct vcpu_svm *svm = to_svm(vcpu);
1025
1026         __free_page(pfn_to_page(svm->vmcb_pa >> PAGE_SHIFT));
1027         __free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
1028         __free_page(virt_to_page(svm->nested.hsave));
1029         __free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER);
1030         kvm_vcpu_uninit(vcpu);
1031         kmem_cache_free(kvm_vcpu_cache, svm);
1032 }
1033
1034 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1035 {
1036         struct vcpu_svm *svm = to_svm(vcpu);
1037         int i;
1038
1039         if (unlikely(cpu != vcpu->cpu)) {
1040                 svm->asid_generation = 0;
1041         }
1042
1043 #ifdef CONFIG_X86_64
1044         rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host.gs_base);
1045 #endif
1046         savesegment(fs, svm->host.fs);
1047         savesegment(gs, svm->host.gs);
1048         svm->host.ldt = kvm_read_ldt();
1049
1050         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
1051                 rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
1052 }
1053
1054 static void svm_vcpu_put(struct kvm_vcpu *vcpu)
1055 {
1056         struct vcpu_svm *svm = to_svm(vcpu);
1057         int i;
1058
1059         ++vcpu->stat.host_state_reload;
1060         kvm_load_ldt(svm->host.ldt);
1061 #ifdef CONFIG_X86_64
1062         loadsegment(fs, svm->host.fs);
1063         load_gs_index(svm->host.gs);
1064         wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gs);
1065 #else
1066         loadsegment(gs, svm->host.gs);
1067 #endif
1068         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
1069                 wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
1070 }
1071
1072 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
1073 {
1074         return to_svm(vcpu)->vmcb->save.rflags;
1075 }
1076
1077 static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
1078 {
1079         to_svm(vcpu)->vmcb->save.rflags = rflags;
1080 }
1081
1082 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
1083 {
1084         switch (reg) {
1085         case VCPU_EXREG_PDPTR:
1086                 BUG_ON(!npt_enabled);
1087                 load_pdptrs(vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3);
1088                 break;
1089         default:
1090                 BUG();
1091         }
1092 }
1093
1094 static void svm_set_vintr(struct vcpu_svm *svm)
1095 {
1096         svm->vmcb->control.intercept |= 1ULL << INTERCEPT_VINTR;
1097 }
1098
1099 static void svm_clear_vintr(struct vcpu_svm *svm)
1100 {
1101         svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VINTR);
1102 }
1103
1104 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
1105 {
1106         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1107
1108         switch (seg) {
1109         case VCPU_SREG_CS: return &save->cs;
1110         case VCPU_SREG_DS: return &save->ds;
1111         case VCPU_SREG_ES: return &save->es;
1112         case VCPU_SREG_FS: return &save->fs;
1113         case VCPU_SREG_GS: return &save->gs;
1114         case VCPU_SREG_SS: return &save->ss;
1115         case VCPU_SREG_TR: return &save->tr;
1116         case VCPU_SREG_LDTR: return &save->ldtr;
1117         }
1118         BUG();
1119         return NULL;
1120 }
1121
1122 static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
1123 {
1124         struct vmcb_seg *s = svm_seg(vcpu, seg);
1125
1126         return s->base;
1127 }
1128
1129 static void svm_get_segment(struct kvm_vcpu *vcpu,
1130                             struct kvm_segment *var, int seg)
1131 {
1132         struct vmcb_seg *s = svm_seg(vcpu, seg);
1133
1134         var->base = s->base;
1135         var->limit = s->limit;
1136         var->selector = s->selector;
1137         var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
1138         var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
1139         var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
1140         var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
1141         var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
1142         var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
1143         var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
1144         var->g = (s->attrib >> SVM_SELECTOR_G_SHIFT) & 1;
1145
1146         /*
1147          * AMD's VMCB does not have an explicit unusable field, so emulate it
1148          * for cross vendor migration purposes by "not present"
1149          */
1150         var->unusable = !var->present || (var->type == 0);
1151
1152         switch (seg) {
1153         case VCPU_SREG_CS:
1154                 /*
1155                  * SVM always stores 0 for the 'G' bit in the CS selector in
1156                  * the VMCB on a VMEXIT. This hurts cross-vendor migration:
1157                  * Intel's VMENTRY has a check on the 'G' bit.
1158                  */
1159                 var->g = s->limit > 0xfffff;
1160                 break;
1161         case VCPU_SREG_TR:
1162                 /*
1163                  * Work around a bug where the busy flag in the tr selector
1164                  * isn't exposed
1165                  */
1166                 var->type |= 0x2;
1167                 break;
1168         case VCPU_SREG_DS:
1169         case VCPU_SREG_ES:
1170         case VCPU_SREG_FS:
1171         case VCPU_SREG_GS:
1172                 /*
1173                  * The accessed bit must always be set in the segment
1174                  * descriptor cache, although it can be cleared in the
1175                  * descriptor, the cached bit always remains at 1. Since
1176                  * Intel has a check on this, set it here to support
1177                  * cross-vendor migration.
1178                  */
1179                 if (!var->unusable)
1180                         var->type |= 0x1;
1181                 break;
1182         case VCPU_SREG_SS:
1183                 /*
1184                  * On AMD CPUs sometimes the DB bit in the segment
1185                  * descriptor is left as 1, although the whole segment has
1186                  * been made unusable. Clear it here to pass an Intel VMX
1187                  * entry check when cross vendor migrating.
1188                  */
1189                 if (var->unusable)
1190                         var->db = 0;
1191                 break;
1192         }
1193 }
1194
1195 static int svm_get_cpl(struct kvm_vcpu *vcpu)
1196 {
1197         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1198
1199         return save->cpl;
1200 }
1201
1202 static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1203 {
1204         struct vcpu_svm *svm = to_svm(vcpu);
1205
1206         dt->size = svm->vmcb->save.idtr.limit;
1207         dt->address = svm->vmcb->save.idtr.base;
1208 }
1209
1210 static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1211 {
1212         struct vcpu_svm *svm = to_svm(vcpu);
1213
1214         svm->vmcb->save.idtr.limit = dt->size;
1215         svm->vmcb->save.idtr.base = dt->address ;
1216 }
1217
1218 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1219 {
1220         struct vcpu_svm *svm = to_svm(vcpu);
1221
1222         dt->size = svm->vmcb->save.gdtr.limit;
1223         dt->address = svm->vmcb->save.gdtr.base;
1224 }
1225
1226 static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1227 {
1228         struct vcpu_svm *svm = to_svm(vcpu);
1229
1230         svm->vmcb->save.gdtr.limit = dt->size;
1231         svm->vmcb->save.gdtr.base = dt->address ;
1232 }
1233
1234 static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
1235 {
1236 }
1237
1238 static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
1239 {
1240 }
1241
1242 static void update_cr0_intercept(struct vcpu_svm *svm)
1243 {
1244         ulong gcr0 = svm->vcpu.arch.cr0;
1245         u64 *hcr0 = &svm->vmcb->save.cr0;
1246
1247         if (!svm->vcpu.fpu_active)
1248                 *hcr0 |= SVM_CR0_SELECTIVE_MASK;
1249         else
1250                 *hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK)
1251                         | (gcr0 & SVM_CR0_SELECTIVE_MASK);
1252
1253
1254         if (gcr0 == *hcr0 && svm->vcpu.fpu_active) {
1255                 clr_cr_intercept(svm, INTERCEPT_CR0_READ);
1256                 clr_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1257         } else {
1258                 set_cr_intercept(svm, INTERCEPT_CR0_READ);
1259                 set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1260         }
1261 }
1262
1263 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1264 {
1265         struct vcpu_svm *svm = to_svm(vcpu);
1266
1267         if (is_guest_mode(vcpu)) {
1268                 /*
1269                  * We are here because we run in nested mode, the host kvm
1270                  * intercepts cr0 writes but the l1 hypervisor does not.
1271                  * But the L1 hypervisor may intercept selective cr0 writes.
1272                  * This needs to be checked here.
1273                  */
1274                 unsigned long old, new;
1275
1276                 /* Remove bits that would trigger a real cr0 write intercept */
1277                 old = vcpu->arch.cr0 & SVM_CR0_SELECTIVE_MASK;
1278                 new = cr0 & SVM_CR0_SELECTIVE_MASK;
1279
1280                 if (old == new) {
1281                         /* cr0 write with ts and mp unchanged */
1282                         svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
1283                         if (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE) {
1284                                 svm->nested.vmexit_rip = kvm_rip_read(vcpu);
1285                                 svm->nested.vmexit_rsp = kvm_register_read(vcpu, VCPU_REGS_RSP);
1286                                 svm->nested.vmexit_rax = kvm_register_read(vcpu, VCPU_REGS_RAX);
1287                                 return;
1288                         }
1289                 }
1290         }
1291
1292 #ifdef CONFIG_X86_64
1293         if (vcpu->arch.efer & EFER_LME) {
1294                 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
1295                         vcpu->arch.efer |= EFER_LMA;
1296                         svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
1297                 }
1298
1299                 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
1300                         vcpu->arch.efer &= ~EFER_LMA;
1301                         svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
1302                 }
1303         }
1304 #endif
1305         vcpu->arch.cr0 = cr0;
1306
1307         if (!npt_enabled)
1308                 cr0 |= X86_CR0_PG | X86_CR0_WP;
1309
1310         if (!vcpu->fpu_active)
1311                 cr0 |= X86_CR0_TS;
1312         /*
1313          * re-enable caching here because the QEMU bios
1314          * does not do it - this results in some delay at
1315          * reboot
1316          */
1317         cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
1318         svm->vmcb->save.cr0 = cr0;
1319         update_cr0_intercept(svm);
1320 }
1321
1322 static void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1323 {
1324         unsigned long host_cr4_mce = read_cr4() & X86_CR4_MCE;
1325         unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;
1326
1327         if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
1328                 force_new_asid(vcpu);
1329
1330         vcpu->arch.cr4 = cr4;
1331         if (!npt_enabled)
1332                 cr4 |= X86_CR4_PAE;
1333         cr4 |= host_cr4_mce;
1334         to_svm(vcpu)->vmcb->save.cr4 = cr4;
1335 }
1336
1337 static void svm_set_segment(struct kvm_vcpu *vcpu,
1338                             struct kvm_segment *var, int seg)
1339 {
1340         struct vcpu_svm *svm = to_svm(vcpu);
1341         struct vmcb_seg *s = svm_seg(vcpu, seg);
1342
1343         s->base = var->base;
1344         s->limit = var->limit;
1345         s->selector = var->selector;
1346         if (var->unusable)
1347                 s->attrib = 0;
1348         else {
1349                 s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
1350                 s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
1351                 s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
1352                 s->attrib |= (var->present & 1) << SVM_SELECTOR_P_SHIFT;
1353                 s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
1354                 s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
1355                 s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
1356                 s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
1357         }
1358         if (seg == VCPU_SREG_CS)
1359                 svm->vmcb->save.cpl
1360                         = (svm->vmcb->save.cs.attrib
1361                            >> SVM_SELECTOR_DPL_SHIFT) & 3;
1362
1363 }
1364
1365 static void update_db_intercept(struct kvm_vcpu *vcpu)
1366 {
1367         struct vcpu_svm *svm = to_svm(vcpu);
1368
1369         svm->vmcb->control.intercept_exceptions &=
1370                 ~((1 << DB_VECTOR) | (1 << BP_VECTOR));
1371
1372         if (svm->nmi_singlestep)
1373                 svm->vmcb->control.intercept_exceptions |= (1 << DB_VECTOR);
1374
1375         if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
1376                 if (vcpu->guest_debug &
1377                     (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
1378                         svm->vmcb->control.intercept_exceptions |=
1379                                 1 << DB_VECTOR;
1380                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
1381                         svm->vmcb->control.intercept_exceptions |=
1382                                 1 << BP_VECTOR;
1383         } else
1384                 vcpu->guest_debug = 0;
1385 }
1386
1387 static void svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg)
1388 {
1389         struct vcpu_svm *svm = to_svm(vcpu);
1390
1391         if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
1392                 svm->vmcb->save.dr7 = dbg->arch.debugreg[7];
1393         else
1394                 svm->vmcb->save.dr7 = vcpu->arch.dr7;
1395
1396         update_db_intercept(vcpu);
1397 }
1398
1399 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
1400 {
1401         if (sd->next_asid > sd->max_asid) {
1402                 ++sd->asid_generation;
1403                 sd->next_asid = 1;
1404                 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
1405         }
1406
1407         svm->asid_generation = sd->asid_generation;
1408         svm->vmcb->control.asid = sd->next_asid++;
1409 }
1410
1411 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
1412 {
1413         struct vcpu_svm *svm = to_svm(vcpu);
1414
1415         svm->vmcb->save.dr7 = value;
1416 }
1417
1418 static int pf_interception(struct vcpu_svm *svm)
1419 {
1420         u64 fault_address = svm->vmcb->control.exit_info_2;
1421         u32 error_code;
1422         int r = 1;
1423
1424         switch (svm->apf_reason) {
1425         default:
1426                 error_code = svm->vmcb->control.exit_info_1;
1427
1428                 trace_kvm_page_fault(fault_address, error_code);
1429                 if (!npt_enabled && kvm_event_needs_reinjection(&svm->vcpu))
1430                         kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address);
1431                 r = kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code);
1432                 break;
1433         case KVM_PV_REASON_PAGE_NOT_PRESENT:
1434                 svm->apf_reason = 0;
1435                 local_irq_disable();
1436                 kvm_async_pf_task_wait(fault_address);
1437                 local_irq_enable();
1438                 break;
1439         case KVM_PV_REASON_PAGE_READY:
1440                 svm->apf_reason = 0;
1441                 local_irq_disable();
1442                 kvm_async_pf_task_wake(fault_address);
1443                 local_irq_enable();
1444                 break;
1445         }
1446         return r;
1447 }
1448
1449 static int db_interception(struct vcpu_svm *svm)
1450 {
1451         struct kvm_run *kvm_run = svm->vcpu.run;
1452
1453         if (!(svm->vcpu.guest_debug &
1454               (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
1455                 !svm->nmi_singlestep) {
1456                 kvm_queue_exception(&svm->vcpu, DB_VECTOR);
1457                 return 1;
1458         }
1459
1460         if (svm->nmi_singlestep) {
1461                 svm->nmi_singlestep = false;
1462                 if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP))
1463                         svm->vmcb->save.rflags &=
1464                                 ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
1465                 update_db_intercept(&svm->vcpu);
1466         }
1467
1468         if (svm->vcpu.guest_debug &
1469             (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
1470                 kvm_run->exit_reason = KVM_EXIT_DEBUG;
1471                 kvm_run->debug.arch.pc =
1472                         svm->vmcb->save.cs.base + svm->vmcb->save.rip;
1473                 kvm_run->debug.arch.exception = DB_VECTOR;
1474                 return 0;
1475         }
1476
1477         return 1;
1478 }
1479
1480 static int bp_interception(struct vcpu_svm *svm)
1481 {
1482         struct kvm_run *kvm_run = svm->vcpu.run;
1483
1484         kvm_run->exit_reason = KVM_EXIT_DEBUG;
1485         kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
1486         kvm_run->debug.arch.exception = BP_VECTOR;
1487         return 0;
1488 }
1489
1490 static int ud_interception(struct vcpu_svm *svm)
1491 {
1492         int er;
1493
1494         er = emulate_instruction(&svm->vcpu, 0, 0, EMULTYPE_TRAP_UD);
1495         if (er != EMULATE_DONE)
1496                 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
1497         return 1;
1498 }
1499
1500 static void svm_fpu_activate(struct kvm_vcpu *vcpu)
1501 {
1502         struct vcpu_svm *svm = to_svm(vcpu);
1503         u32 excp;
1504
1505         if (is_guest_mode(vcpu)) {
1506                 u32 h_excp, n_excp;
1507
1508                 h_excp  = svm->nested.hsave->control.intercept_exceptions;
1509                 n_excp  = svm->nested.intercept_exceptions;
1510                 h_excp &= ~(1 << NM_VECTOR);
1511                 excp    = h_excp | n_excp;
1512         } else {
1513                 excp  = svm->vmcb->control.intercept_exceptions;
1514                 excp &= ~(1 << NM_VECTOR);
1515         }
1516
1517         svm->vmcb->control.intercept_exceptions = excp;
1518
1519         svm->vcpu.fpu_active = 1;
1520         update_cr0_intercept(svm);
1521 }
1522
1523 static int nm_interception(struct vcpu_svm *svm)
1524 {
1525         svm_fpu_activate(&svm->vcpu);
1526         return 1;
1527 }
1528
1529 static bool is_erratum_383(void)
1530 {
1531         int err, i;
1532         u64 value;
1533
1534         if (!erratum_383_found)
1535                 return false;
1536
1537         value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
1538         if (err)
1539                 return false;
1540
1541         /* Bit 62 may or may not be set for this mce */
1542         value &= ~(1ULL << 62);
1543
1544         if (value != 0xb600000000010015ULL)
1545                 return false;
1546
1547         /* Clear MCi_STATUS registers */
1548         for (i = 0; i < 6; ++i)
1549                 native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);
1550
1551         value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
1552         if (!err) {
1553                 u32 low, high;
1554
1555                 value &= ~(1ULL << 2);
1556                 low    = lower_32_bits(value);
1557                 high   = upper_32_bits(value);
1558
1559                 native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
1560         }
1561
1562         /* Flush tlb to evict multi-match entries */
1563         __flush_tlb_all();
1564
1565         return true;
1566 }
1567
1568 static void svm_handle_mce(struct vcpu_svm *svm)
1569 {
1570         if (is_erratum_383()) {
1571                 /*
1572                  * Erratum 383 triggered. Guest state is corrupt so kill the
1573                  * guest.
1574                  */
1575                 pr_err("KVM: Guest triggered AMD Erratum 383\n");
1576
1577                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, &svm->vcpu);
1578
1579                 return;
1580         }
1581
1582         /*
1583          * On an #MC intercept the MCE handler is not called automatically in
1584          * the host. So do it by hand here.
1585          */
1586         asm volatile (
1587                 "int $0x12\n");
1588         /* not sure if we ever come back to this point */
1589
1590         return;
1591 }
1592
1593 static int mc_interception(struct vcpu_svm *svm)
1594 {
1595         return 1;
1596 }
1597
1598 static int shutdown_interception(struct vcpu_svm *svm)
1599 {
1600         struct kvm_run *kvm_run = svm->vcpu.run;
1601
1602         /*
1603          * VMCB is undefined after a SHUTDOWN intercept
1604          * so reinitialize it.
1605          */
1606         clear_page(svm->vmcb);
1607         init_vmcb(svm);
1608
1609         kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
1610         return 0;
1611 }
1612
1613 static int io_interception(struct vcpu_svm *svm)
1614 {
1615         struct kvm_vcpu *vcpu = &svm->vcpu;
1616         u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
1617         int size, in, string;
1618         unsigned port;
1619
1620         ++svm->vcpu.stat.io_exits;
1621         string = (io_info & SVM_IOIO_STR_MASK) != 0;
1622         in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
1623         if (string || in)
1624                 return emulate_instruction(vcpu, 0, 0, 0) == EMULATE_DONE;
1625
1626         port = io_info >> 16;
1627         size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
1628         svm->next_rip = svm->vmcb->control.exit_info_2;
1629         skip_emulated_instruction(&svm->vcpu);
1630
1631         return kvm_fast_pio_out(vcpu, size, port);
1632 }
1633
1634 static int nmi_interception(struct vcpu_svm *svm)
1635 {
1636         return 1;
1637 }
1638
1639 static int intr_interception(struct vcpu_svm *svm)
1640 {
1641         ++svm->vcpu.stat.irq_exits;
1642         return 1;
1643 }
1644
1645 static int nop_on_interception(struct vcpu_svm *svm)
1646 {
1647         return 1;
1648 }
1649
1650 static int halt_interception(struct vcpu_svm *svm)
1651 {
1652         svm->next_rip = kvm_rip_read(&svm->vcpu) + 1;
1653         skip_emulated_instruction(&svm->vcpu);
1654         return kvm_emulate_halt(&svm->vcpu);
1655 }
1656
1657 static int vmmcall_interception(struct vcpu_svm *svm)
1658 {
1659         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
1660         skip_emulated_instruction(&svm->vcpu);
1661         kvm_emulate_hypercall(&svm->vcpu);
1662         return 1;
1663 }
1664
1665 static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu)
1666 {
1667         struct vcpu_svm *svm = to_svm(vcpu);
1668
1669         return svm->nested.nested_cr3;
1670 }
1671
1672 static void nested_svm_set_tdp_cr3(struct kvm_vcpu *vcpu,
1673                                    unsigned long root)
1674 {
1675         struct vcpu_svm *svm = to_svm(vcpu);
1676
1677         svm->vmcb->control.nested_cr3 = root;
1678         force_new_asid(vcpu);
1679 }
1680
1681 static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
1682                                        struct x86_exception *fault)
1683 {
1684         struct vcpu_svm *svm = to_svm(vcpu);
1685
1686         svm->vmcb->control.exit_code = SVM_EXIT_NPF;
1687         svm->vmcb->control.exit_code_hi = 0;
1688         svm->vmcb->control.exit_info_1 = fault->error_code;
1689         svm->vmcb->control.exit_info_2 = fault->address;
1690
1691         nested_svm_vmexit(svm);
1692 }
1693
1694 static int nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
1695 {
1696         int r;
1697
1698         r = kvm_init_shadow_mmu(vcpu, &vcpu->arch.mmu);
1699
1700         vcpu->arch.mmu.set_cr3           = nested_svm_set_tdp_cr3;
1701         vcpu->arch.mmu.get_cr3           = nested_svm_get_tdp_cr3;
1702         vcpu->arch.mmu.inject_page_fault = nested_svm_inject_npf_exit;
1703         vcpu->arch.mmu.shadow_root_level = get_npt_level();
1704         vcpu->arch.walk_mmu              = &vcpu->arch.nested_mmu;
1705
1706         return r;
1707 }
1708
1709 static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
1710 {
1711         vcpu->arch.walk_mmu = &vcpu->arch.mmu;
1712 }
1713
1714 static int nested_svm_check_permissions(struct vcpu_svm *svm)
1715 {
1716         if (!(svm->vcpu.arch.efer & EFER_SVME)
1717             || !is_paging(&svm->vcpu)) {
1718                 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
1719                 return 1;
1720         }
1721
1722         if (svm->vmcb->save.cpl) {
1723                 kvm_inject_gp(&svm->vcpu, 0);
1724                 return 1;
1725         }
1726
1727        return 0;
1728 }
1729
1730 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
1731                                       bool has_error_code, u32 error_code)
1732 {
1733         int vmexit;
1734
1735         if (!is_guest_mode(&svm->vcpu))
1736                 return 0;
1737
1738         svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
1739         svm->vmcb->control.exit_code_hi = 0;
1740         svm->vmcb->control.exit_info_1 = error_code;
1741         svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;
1742
1743         vmexit = nested_svm_intercept(svm);
1744         if (vmexit == NESTED_EXIT_DONE)
1745                 svm->nested.exit_required = true;
1746
1747         return vmexit;
1748 }
1749
1750 /* This function returns true if it is save to enable the irq window */
1751 static inline bool nested_svm_intr(struct vcpu_svm *svm)
1752 {
1753         if (!is_guest_mode(&svm->vcpu))
1754                 return true;
1755
1756         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
1757                 return true;
1758
1759         if (!(svm->vcpu.arch.hflags & HF_HIF_MASK))
1760                 return false;
1761
1762         /*
1763          * if vmexit was already requested (by intercepted exception
1764          * for instance) do not overwrite it with "external interrupt"
1765          * vmexit.
1766          */
1767         if (svm->nested.exit_required)
1768                 return false;
1769
1770         svm->vmcb->control.exit_code   = SVM_EXIT_INTR;
1771         svm->vmcb->control.exit_info_1 = 0;
1772         svm->vmcb->control.exit_info_2 = 0;
1773
1774         if (svm->nested.intercept & 1ULL) {
1775                 /*
1776                  * The #vmexit can't be emulated here directly because this
1777                  * code path runs with irqs and preemtion disabled. A
1778                  * #vmexit emulation might sleep. Only signal request for
1779                  * the #vmexit here.
1780                  */
1781                 svm->nested.exit_required = true;
1782                 trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);
1783                 return false;
1784         }
1785
1786         return true;
1787 }
1788
1789 /* This function returns true if it is save to enable the nmi window */
1790 static inline bool nested_svm_nmi(struct vcpu_svm *svm)
1791 {
1792         if (!is_guest_mode(&svm->vcpu))
1793                 return true;
1794
1795         if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI)))
1796                 return true;
1797
1798         svm->vmcb->control.exit_code = SVM_EXIT_NMI;
1799         svm->nested.exit_required = true;
1800
1801         return false;
1802 }
1803
1804 static void *nested_svm_map(struct vcpu_svm *svm, u64 gpa, struct page **_page)
1805 {
1806         struct page *page;
1807
1808         might_sleep();
1809
1810         page = gfn_to_page(svm->vcpu.kvm, gpa >> PAGE_SHIFT);
1811         if (is_error_page(page))
1812                 goto error;
1813
1814         *_page = page;
1815
1816         return kmap(page);
1817
1818 error:
1819         kvm_release_page_clean(page);
1820         kvm_inject_gp(&svm->vcpu, 0);
1821
1822         return NULL;
1823 }
1824
1825 static void nested_svm_unmap(struct page *page)
1826 {
1827         kunmap(page);
1828         kvm_release_page_dirty(page);
1829 }
1830
1831 static int nested_svm_intercept_ioio(struct vcpu_svm *svm)
1832 {
1833         unsigned port;
1834         u8 val, bit;
1835         u64 gpa;
1836
1837         if (!(svm->nested.intercept & (1ULL << INTERCEPT_IOIO_PROT)))
1838                 return NESTED_EXIT_HOST;
1839
1840         port = svm->vmcb->control.exit_info_1 >> 16;
1841         gpa  = svm->nested.vmcb_iopm + (port / 8);
1842         bit  = port % 8;
1843         val  = 0;
1844
1845         if (kvm_read_guest(svm->vcpu.kvm, gpa, &val, 1))
1846                 val &= (1 << bit);
1847
1848         return val ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
1849 }
1850
1851 static int nested_svm_exit_handled_msr(struct vcpu_svm *svm)
1852 {
1853         u32 offset, msr, value;
1854         int write, mask;
1855
1856         if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
1857                 return NESTED_EXIT_HOST;
1858
1859         msr    = svm->vcpu.arch.regs[VCPU_REGS_RCX];
1860         offset = svm_msrpm_offset(msr);
1861         write  = svm->vmcb->control.exit_info_1 & 1;
1862         mask   = 1 << ((2 * (msr & 0xf)) + write);
1863
1864         if (offset == MSR_INVALID)
1865                 return NESTED_EXIT_DONE;
1866
1867         /* Offset is in 32 bit units but need in 8 bit units */
1868         offset *= 4;
1869
1870         if (kvm_read_guest(svm->vcpu.kvm, svm->nested.vmcb_msrpm + offset, &value, 4))
1871                 return NESTED_EXIT_DONE;
1872
1873         return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
1874 }
1875
1876 static int nested_svm_exit_special(struct vcpu_svm *svm)
1877 {
1878         u32 exit_code = svm->vmcb->control.exit_code;
1879
1880         switch (exit_code) {
1881         case SVM_EXIT_INTR:
1882         case SVM_EXIT_NMI:
1883         case SVM_EXIT_EXCP_BASE + MC_VECTOR:
1884                 return NESTED_EXIT_HOST;
1885         case SVM_EXIT_NPF:
1886                 /* For now we are always handling NPFs when using them */
1887                 if (npt_enabled)
1888                         return NESTED_EXIT_HOST;
1889                 break;
1890         case SVM_EXIT_EXCP_BASE + PF_VECTOR:
1891                 /* When we're shadowing, trap PFs, but not async PF */
1892                 if (!npt_enabled && svm->apf_reason == 0)
1893                         return NESTED_EXIT_HOST;
1894                 break;
1895         case SVM_EXIT_EXCP_BASE + NM_VECTOR:
1896                 nm_interception(svm);
1897                 break;
1898         default:
1899                 break;
1900         }
1901
1902         return NESTED_EXIT_CONTINUE;
1903 }
1904
1905 /*
1906  * If this function returns true, this #vmexit was already handled
1907  */
1908 static int nested_svm_intercept(struct vcpu_svm *svm)
1909 {
1910         u32 exit_code = svm->vmcb->control.exit_code;
1911         int vmexit = NESTED_EXIT_HOST;
1912
1913         switch (exit_code) {
1914         case SVM_EXIT_MSR:
1915                 vmexit = nested_svm_exit_handled_msr(svm);
1916                 break;
1917         case SVM_EXIT_IOIO:
1918                 vmexit = nested_svm_intercept_ioio(svm);
1919                 break;
1920         case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: {
1921                 u32 bit = 1U << (exit_code - SVM_EXIT_READ_CR0);
1922                 if (svm->nested.intercept_cr & bit)
1923                         vmexit = NESTED_EXIT_DONE;
1924                 break;
1925         }
1926         case SVM_EXIT_READ_DR0 ... SVM_EXIT_READ_DR7: {
1927                 u32 dr_bits = 1 << (exit_code - SVM_EXIT_READ_DR0);
1928                 if (svm->nested.intercept_dr_read & dr_bits)
1929                         vmexit = NESTED_EXIT_DONE;
1930                 break;
1931         }
1932         case SVM_EXIT_WRITE_DR0 ... SVM_EXIT_WRITE_DR7: {
1933                 u32 dr_bits = 1 << (exit_code - SVM_EXIT_WRITE_DR0);
1934                 if (svm->nested.intercept_dr_write & dr_bits)
1935                         vmexit = NESTED_EXIT_DONE;
1936                 break;
1937         }
1938         case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
1939                 u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
1940                 if (svm->nested.intercept_exceptions & excp_bits)
1941                         vmexit = NESTED_EXIT_DONE;
1942                 /* async page fault always cause vmexit */
1943                 else if ((exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) &&
1944                          svm->apf_reason != 0)
1945                         vmexit = NESTED_EXIT_DONE;
1946                 break;
1947         }
1948         case SVM_EXIT_ERR: {
1949                 vmexit = NESTED_EXIT_DONE;
1950                 break;
1951         }
1952         default: {
1953                 u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR);
1954                 if (svm->nested.intercept & exit_bits)
1955                         vmexit = NESTED_EXIT_DONE;
1956         }
1957         }
1958
1959         return vmexit;
1960 }
1961
1962 static int nested_svm_exit_handled(struct vcpu_svm *svm)
1963 {
1964         int vmexit;
1965
1966         vmexit = nested_svm_intercept(svm);
1967
1968         if (vmexit == NESTED_EXIT_DONE)
1969                 nested_svm_vmexit(svm);
1970
1971         return vmexit;
1972 }
1973
1974 static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb)
1975 {
1976         struct vmcb_control_area *dst  = &dst_vmcb->control;
1977         struct vmcb_control_area *from = &from_vmcb->control;
1978
1979         dst->intercept_cr         = from->intercept_cr;
1980         dst->intercept_dr_read    = from->intercept_dr_read;
1981         dst->intercept_dr_write   = from->intercept_dr_write;
1982         dst->intercept_exceptions = from->intercept_exceptions;
1983         dst->intercept            = from->intercept;
1984         dst->iopm_base_pa         = from->iopm_base_pa;
1985         dst->msrpm_base_pa        = from->msrpm_base_pa;
1986         dst->tsc_offset           = from->tsc_offset;
1987         dst->asid                 = from->asid;
1988         dst->tlb_ctl              = from->tlb_ctl;
1989         dst->int_ctl              = from->int_ctl;
1990         dst->int_vector           = from->int_vector;
1991         dst->int_state            = from->int_state;
1992         dst->exit_code            = from->exit_code;
1993         dst->exit_code_hi         = from->exit_code_hi;
1994         dst->exit_info_1          = from->exit_info_1;
1995         dst->exit_info_2          = from->exit_info_2;
1996         dst->exit_int_info        = from->exit_int_info;
1997         dst->exit_int_info_err    = from->exit_int_info_err;
1998         dst->nested_ctl           = from->nested_ctl;
1999         dst->event_inj            = from->event_inj;
2000         dst->event_inj_err        = from->event_inj_err;
2001         dst->nested_cr3           = from->nested_cr3;
2002         dst->lbr_ctl              = from->lbr_ctl;
2003 }
2004
2005 static int nested_svm_vmexit(struct vcpu_svm *svm)
2006 {
2007         struct vmcb *nested_vmcb;
2008         struct vmcb *hsave = svm->nested.hsave;
2009         struct vmcb *vmcb = svm->vmcb;
2010         struct page *page;
2011
2012         trace_kvm_nested_vmexit_inject(vmcb->control.exit_code,
2013                                        vmcb->control.exit_info_1,
2014                                        vmcb->control.exit_info_2,
2015                                        vmcb->control.exit_int_info,
2016                                        vmcb->control.exit_int_info_err);
2017
2018         nested_vmcb = nested_svm_map(svm, svm->nested.vmcb, &page);
2019         if (!nested_vmcb)
2020                 return 1;
2021
2022         /* Exit Guest-Mode */
2023         leave_guest_mode(&svm->vcpu);
2024         svm->nested.vmcb = 0;
2025
2026         /* Give the current vmcb to the guest */
2027         disable_gif(svm);
2028
2029         nested_vmcb->save.es     = vmcb->save.es;
2030         nested_vmcb->save.cs     = vmcb->save.cs;
2031         nested_vmcb->save.ss     = vmcb->save.ss;
2032         nested_vmcb->save.ds     = vmcb->save.ds;
2033         nested_vmcb->save.gdtr   = vmcb->save.gdtr;
2034         nested_vmcb->save.idtr   = vmcb->save.idtr;
2035         nested_vmcb->save.efer   = svm->vcpu.arch.efer;
2036         nested_vmcb->save.cr0    = kvm_read_cr0(&svm->vcpu);
2037         nested_vmcb->save.cr3    = svm->vcpu.arch.cr3;
2038         nested_vmcb->save.cr2    = vmcb->save.cr2;
2039         nested_vmcb->save.cr4    = svm->vcpu.arch.cr4;
2040         nested_vmcb->save.rflags = vmcb->save.rflags;
2041         nested_vmcb->save.rip    = vmcb->save.rip;
2042         nested_vmcb->save.rsp    = vmcb->save.rsp;
2043         nested_vmcb->save.rax    = vmcb->save.rax;
2044         nested_vmcb->save.dr7    = vmcb->save.dr7;
2045         nested_vmcb->save.dr6    = vmcb->save.dr6;
2046         nested_vmcb->save.cpl    = vmcb->save.cpl;
2047
2048         nested_vmcb->control.int_ctl           = vmcb->control.int_ctl;
2049         nested_vmcb->control.int_vector        = vmcb->control.int_vector;
2050         nested_vmcb->control.int_state         = vmcb->control.int_state;
2051         nested_vmcb->control.exit_code         = vmcb->control.exit_code;
2052         nested_vmcb->control.exit_code_hi      = vmcb->control.exit_code_hi;
2053         nested_vmcb->control.exit_info_1       = vmcb->control.exit_info_1;
2054         nested_vmcb->control.exit_info_2       = vmcb->control.exit_info_2;
2055         nested_vmcb->control.exit_int_info     = vmcb->control.exit_int_info;
2056         nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err;
2057         nested_vmcb->control.next_rip          = vmcb->control.next_rip;
2058
2059         /*
2060          * If we emulate a VMRUN/#VMEXIT in the same host #vmexit cycle we have
2061          * to make sure that we do not lose injected events. So check event_inj
2062          * here and copy it to exit_int_info if it is valid.
2063          * Exit_int_info and event_inj can't be both valid because the case
2064          * below only happens on a VMRUN instruction intercept which has
2065          * no valid exit_int_info set.
2066          */
2067         if (vmcb->control.event_inj & SVM_EVTINJ_VALID) {
2068                 struct vmcb_control_area *nc = &nested_vmcb->control;
2069
2070                 nc->exit_int_info     = vmcb->control.event_inj;
2071                 nc->exit_int_info_err = vmcb->control.event_inj_err;
2072         }
2073
2074         nested_vmcb->control.tlb_ctl           = 0;
2075         nested_vmcb->control.event_inj         = 0;
2076         nested_vmcb->control.event_inj_err     = 0;
2077
2078         /* We always set V_INTR_MASKING and remember the old value in hflags */
2079         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
2080                 nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
2081
2082         /* Restore the original control entries */
2083         copy_vmcb_control_area(vmcb, hsave);
2084
2085         kvm_clear_exception_queue(&svm->vcpu);
2086         kvm_clear_interrupt_queue(&svm->vcpu);
2087
2088         svm->nested.nested_cr3 = 0;
2089
2090         /* Restore selected save entries */
2091         svm->vmcb->save.es = hsave->save.es;
2092         svm->vmcb->save.cs = hsave->save.cs;
2093         svm->vmcb->save.ss = hsave->save.ss;
2094         svm->vmcb->save.ds = hsave->save.ds;
2095         svm->vmcb->save.gdtr = hsave->save.gdtr;
2096         svm->vmcb->save.idtr = hsave->save.idtr;
2097         svm->vmcb->save.rflags = hsave->save.rflags;
2098         svm_set_efer(&svm->vcpu, hsave->save.efer);
2099         svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE);
2100         svm_set_cr4(&svm->vcpu, hsave->save.cr4);
2101         if (npt_enabled) {
2102                 svm->vmcb->save.cr3 = hsave->save.cr3;
2103                 svm->vcpu.arch.cr3 = hsave->save.cr3;
2104         } else {
2105                 (void)kvm_set_cr3(&svm->vcpu, hsave->save.cr3);
2106         }
2107         kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, hsave->save.rax);
2108         kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, hsave->save.rsp);
2109         kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, hsave->save.rip);
2110         svm->vmcb->save.dr7 = 0;
2111         svm->vmcb->save.cpl = 0;
2112         svm->vmcb->control.exit_int_info = 0;
2113
2114         nested_svm_unmap(page);
2115
2116         nested_svm_uninit_mmu_context(&svm->vcpu);
2117         kvm_mmu_reset_context(&svm->vcpu);
2118         kvm_mmu_load(&svm->vcpu);
2119
2120         return 0;
2121 }
2122
2123 static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
2124 {
2125         /*
2126          * This function merges the msr permission bitmaps of kvm and the
2127          * nested vmcb. It is omptimized in that it only merges the parts where
2128          * the kvm msr permission bitmap may contain zero bits
2129          */
2130         int i;
2131
2132         if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
2133                 return true;
2134
2135         for (i = 0; i < MSRPM_OFFSETS; i++) {
2136                 u32 value, p;
2137                 u64 offset;
2138
2139                 if (msrpm_offsets[i] == 0xffffffff)
2140                         break;
2141
2142                 p      = msrpm_offsets[i];
2143                 offset = svm->nested.vmcb_msrpm + (p * 4);
2144
2145                 if (kvm_read_guest(svm->vcpu.kvm, offset, &value, 4))
2146                         return false;
2147
2148                 svm->nested.msrpm[p] = svm->msrpm[p] | value;
2149         }
2150
2151         svm->vmcb->control.msrpm_base_pa = __pa(svm->nested.msrpm);
2152
2153         return true;
2154 }
2155
2156 static bool nested_vmcb_checks(struct vmcb *vmcb)
2157 {
2158         if ((vmcb->control.intercept & (1ULL << INTERCEPT_VMRUN)) == 0)
2159                 return false;
2160
2161         if (vmcb->control.asid == 0)
2162                 return false;
2163
2164         if (vmcb->control.nested_ctl && !npt_enabled)
2165                 return false;
2166
2167         return true;
2168 }
2169
2170 static bool nested_svm_vmrun(struct vcpu_svm *svm)
2171 {
2172         struct vmcb *nested_vmcb;
2173         struct vmcb *hsave = svm->nested.hsave;
2174         struct vmcb *vmcb = svm->vmcb;
2175         struct page *page;
2176         u64 vmcb_gpa;
2177
2178         vmcb_gpa = svm->vmcb->save.rax;
2179
2180         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
2181         if (!nested_vmcb)
2182                 return false;
2183
2184         if (!nested_vmcb_checks(nested_vmcb)) {
2185                 nested_vmcb->control.exit_code    = SVM_EXIT_ERR;
2186                 nested_vmcb->control.exit_code_hi = 0;
2187                 nested_vmcb->control.exit_info_1  = 0;
2188                 nested_vmcb->control.exit_info_2  = 0;
2189
2190                 nested_svm_unmap(page);
2191
2192                 return false;
2193         }
2194
2195         trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb_gpa,
2196                                nested_vmcb->save.rip,
2197                                nested_vmcb->control.int_ctl,
2198                                nested_vmcb->control.event_inj,
2199                                nested_vmcb->control.nested_ctl);
2200
2201         trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr & 0xffff,
2202                                     nested_vmcb->control.intercept_cr >> 16,
2203                                     nested_vmcb->control.intercept_exceptions,
2204                                     nested_vmcb->control.intercept);
2205
2206         /* Clear internal status */
2207         kvm_clear_exception_queue(&svm->vcpu);
2208         kvm_clear_interrupt_queue(&svm->vcpu);
2209
2210         /*
2211          * Save the old vmcb, so we don't need to pick what we save, but can
2212          * restore everything when a VMEXIT occurs
2213          */
2214         hsave->save.es     = vmcb->save.es;
2215         hsave->save.cs     = vmcb->save.cs;
2216         hsave->save.ss     = vmcb->save.ss;
2217         hsave->save.ds     = vmcb->save.ds;
2218         hsave->save.gdtr   = vmcb->save.gdtr;
2219         hsave->save.idtr   = vmcb->save.idtr;
2220         hsave->save.efer   = svm->vcpu.arch.efer;
2221         hsave->save.cr0    = kvm_read_cr0(&svm->vcpu);
2222         hsave->save.cr4    = svm->vcpu.arch.cr4;
2223         hsave->save.rflags = vmcb->save.rflags;
2224         hsave->save.rip    = kvm_rip_read(&svm->vcpu);
2225         hsave->save.rsp    = vmcb->save.rsp;
2226         hsave->save.rax    = vmcb->save.rax;
2227         if (npt_enabled)
2228                 hsave->save.cr3    = vmcb->save.cr3;
2229         else
2230                 hsave->save.cr3    = svm->vcpu.arch.cr3;
2231
2232         copy_vmcb_control_area(hsave, vmcb);
2233
2234         if (svm->vmcb->save.rflags & X86_EFLAGS_IF)
2235                 svm->vcpu.arch.hflags |= HF_HIF_MASK;
2236         else
2237                 svm->vcpu.arch.hflags &= ~HF_HIF_MASK;
2238
2239         if (nested_vmcb->control.nested_ctl) {
2240                 kvm_mmu_unload(&svm->vcpu);
2241                 svm->nested.nested_cr3 = nested_vmcb->control.nested_cr3;
2242                 nested_svm_init_mmu_context(&svm->vcpu);
2243         }
2244
2245         /* Load the nested guest state */
2246         svm->vmcb->save.es = nested_vmcb->save.es;
2247         svm->vmcb->save.cs = nested_vmcb->save.cs;
2248         svm->vmcb->save.ss = nested_vmcb->save.ss;
2249         svm->vmcb->save.ds = nested_vmcb->save.ds;
2250         svm->vmcb->save.gdtr = nested_vmcb->save.gdtr;
2251         svm->vmcb->save.idtr = nested_vmcb->save.idtr;
2252         svm->vmcb->save.rflags = nested_vmcb->save.rflags;
2253         svm_set_efer(&svm->vcpu, nested_vmcb->save.efer);
2254         svm_set_cr0(&svm->vcpu, nested_vmcb->save.cr0);
2255         svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4);
2256         if (npt_enabled) {
2257                 svm->vmcb->save.cr3 = nested_vmcb->save.cr3;
2258                 svm->vcpu.arch.cr3 = nested_vmcb->save.cr3;
2259         } else
2260                 (void)kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3);
2261
2262         /* Guest paging mode is active - reset mmu */
2263         kvm_mmu_reset_context(&svm->vcpu);
2264
2265         svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2;
2266         kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, nested_vmcb->save.rax);
2267         kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, nested_vmcb->save.rsp);
2268         kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, nested_vmcb->save.rip);
2269
2270         /* In case we don't even reach vcpu_run, the fields are not updated */
2271         svm->vmcb->save.rax = nested_vmcb->save.rax;
2272         svm->vmcb->save.rsp = nested_vmcb->save.rsp;
2273         svm->vmcb->save.rip = nested_vmcb->save.rip;
2274         svm->vmcb->save.dr7 = nested_vmcb->save.dr7;
2275         svm->vmcb->save.dr6 = nested_vmcb->save.dr6;
2276         svm->vmcb->save.cpl = nested_vmcb->save.cpl;
2277
2278         svm->nested.vmcb_msrpm = nested_vmcb->control.msrpm_base_pa & ~0x0fffULL;
2279         svm->nested.vmcb_iopm  = nested_vmcb->control.iopm_base_pa  & ~0x0fffULL;
2280
2281         /* cache intercepts */
2282         svm->nested.intercept_cr         = nested_vmcb->control.intercept_cr;
2283         svm->nested.intercept_dr_read    = nested_vmcb->control.intercept_dr_read;
2284         svm->nested.intercept_dr_write   = nested_vmcb->control.intercept_dr_write;
2285         svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions;
2286         svm->nested.intercept            = nested_vmcb->control.intercept;
2287
2288         force_new_asid(&svm->vcpu);
2289         svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK;
2290         if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK)
2291                 svm->vcpu.arch.hflags |= HF_VINTR_MASK;
2292         else
2293                 svm->vcpu.arch.hflags &= ~HF_VINTR_MASK;
2294
2295         if (svm->vcpu.arch.hflags & HF_VINTR_MASK) {
2296                 /* We only want the cr8 intercept bits of the guest */
2297                 clr_cr_intercept(svm, INTERCEPT_CR8_READ);
2298                 clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
2299         }
2300
2301         /* We don't want to see VMMCALLs from a nested guest */
2302         svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VMMCALL);
2303
2304         svm->vmcb->control.lbr_ctl = nested_vmcb->control.lbr_ctl;
2305         svm->vmcb->control.int_vector = nested_vmcb->control.int_vector;
2306         svm->vmcb->control.int_state = nested_vmcb->control.int_state;
2307         svm->vmcb->control.tsc_offset += nested_vmcb->control.tsc_offset;
2308         svm->vmcb->control.event_inj = nested_vmcb->control.event_inj;
2309         svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err;
2310
2311         nested_svm_unmap(page);
2312
2313         /* Enter Guest-Mode */
2314         enter_guest_mode(&svm->vcpu);
2315
2316         /*
2317          * Merge guest and host intercepts - must be called  with vcpu in
2318          * guest-mode to take affect here
2319          */
2320         recalc_intercepts(svm);
2321
2322         svm->nested.vmcb = vmcb_gpa;
2323
2324         enable_gif(svm);
2325
2326         return true;
2327 }
2328
2329 static void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
2330 {
2331         to_vmcb->save.fs = from_vmcb->save.fs;
2332         to_vmcb->save.gs = from_vmcb->save.gs;
2333         to_vmcb->save.tr = from_vmcb->save.tr;
2334         to_vmcb->save.ldtr = from_vmcb->save.ldtr;
2335         to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base;
2336         to_vmcb->save.star = from_vmcb->save.star;
2337         to_vmcb->save.lstar = from_vmcb->save.lstar;
2338         to_vmcb->save.cstar = from_vmcb->save.cstar;
2339         to_vmcb->save.sfmask = from_vmcb->save.sfmask;
2340         to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
2341         to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
2342         to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
2343 }
2344
2345 static int vmload_interception(struct vcpu_svm *svm)
2346 {
2347         struct vmcb *nested_vmcb;
2348         struct page *page;
2349
2350         if (nested_svm_check_permissions(svm))
2351                 return 1;
2352
2353         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2354         skip_emulated_instruction(&svm->vcpu);
2355
2356         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
2357         if (!nested_vmcb)
2358                 return 1;
2359
2360         nested_svm_vmloadsave(nested_vmcb, svm->vmcb);
2361         nested_svm_unmap(page);
2362
2363         return 1;
2364 }
2365
2366 static int vmsave_interception(struct vcpu_svm *svm)
2367 {
2368         struct vmcb *nested_vmcb;
2369         struct page *page;
2370
2371         if (nested_svm_check_permissions(svm))
2372                 return 1;
2373
2374         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2375         skip_emulated_instruction(&svm->vcpu);
2376
2377         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
2378         if (!nested_vmcb)
2379                 return 1;
2380
2381         nested_svm_vmloadsave(svm->vmcb, nested_vmcb);
2382         nested_svm_unmap(page);
2383
2384         return 1;
2385 }
2386
2387 static int vmrun_interception(struct vcpu_svm *svm)
2388 {
2389         if (nested_svm_check_permissions(svm))
2390                 return 1;
2391
2392         /* Save rip after vmrun instruction */
2393         kvm_rip_write(&svm->vcpu, kvm_rip_read(&svm->vcpu) + 3);
2394
2395         if (!nested_svm_vmrun(svm))
2396                 return 1;
2397
2398         if (!nested_svm_vmrun_msrpm(svm))
2399                 goto failed;
2400
2401         return 1;
2402
2403 failed:
2404
2405         svm->vmcb->control.exit_code    = SVM_EXIT_ERR;
2406         svm->vmcb->control.exit_code_hi = 0;
2407         svm->vmcb->control.exit_info_1  = 0;
2408         svm->vmcb->control.exit_info_2  = 0;
2409
2410         nested_svm_vmexit(svm);
2411
2412         return 1;
2413 }
2414
2415 static int stgi_interception(struct vcpu_svm *svm)
2416 {
2417         if (nested_svm_check_permissions(svm))
2418                 return 1;
2419
2420         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2421         skip_emulated_instruction(&svm->vcpu);
2422         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2423
2424         enable_gif(svm);
2425
2426         return 1;
2427 }
2428
2429 static int clgi_interception(struct vcpu_svm *svm)
2430 {
2431         if (nested_svm_check_permissions(svm))
2432                 return 1;
2433
2434         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2435         skip_emulated_instruction(&svm->vcpu);
2436
2437         disable_gif(svm);
2438
2439         /* After a CLGI no interrupts should come */
2440         svm_clear_vintr(svm);
2441         svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
2442
2443         return 1;
2444 }
2445
2446 static int invlpga_interception(struct vcpu_svm *svm)
2447 {
2448         struct kvm_vcpu *vcpu = &svm->vcpu;
2449
2450         trace_kvm_invlpga(svm->vmcb->save.rip, vcpu->arch.regs[VCPU_REGS_RCX],
2451                           vcpu->arch.regs[VCPU_REGS_RAX]);
2452
2453         /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
2454         kvm_mmu_invlpg(vcpu, vcpu->arch.regs[VCPU_REGS_RAX]);
2455
2456         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2457         skip_emulated_instruction(&svm->vcpu);
2458         return 1;
2459 }
2460
2461 static int skinit_interception(struct vcpu_svm *svm)
2462 {
2463         trace_kvm_skinit(svm->vmcb->save.rip, svm->vcpu.arch.regs[VCPU_REGS_RAX]);
2464
2465         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2466         return 1;
2467 }
2468
2469 static int invalid_op_interception(struct vcpu_svm *svm)
2470 {
2471         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2472         return 1;
2473 }
2474
2475 static int task_switch_interception(struct vcpu_svm *svm)
2476 {
2477         u16 tss_selector;
2478         int reason;
2479         int int_type = svm->vmcb->control.exit_int_info &
2480                 SVM_EXITINTINFO_TYPE_MASK;
2481         int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
2482         uint32_t type =
2483                 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
2484         uint32_t idt_v =
2485                 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
2486         bool has_error_code = false;
2487         u32 error_code = 0;
2488
2489         tss_selector = (u16)svm->vmcb->control.exit_info_1;
2490
2491         if (svm->vmcb->control.exit_info_2 &
2492             (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
2493                 reason = TASK_SWITCH_IRET;
2494         else if (svm->vmcb->control.exit_info_2 &
2495                  (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
2496                 reason = TASK_SWITCH_JMP;
2497         else if (idt_v)
2498                 reason = TASK_SWITCH_GATE;
2499         else
2500                 reason = TASK_SWITCH_CALL;
2501
2502         if (reason == TASK_SWITCH_GATE) {
2503                 switch (type) {
2504                 case SVM_EXITINTINFO_TYPE_NMI:
2505                         svm->vcpu.arch.nmi_injected = false;
2506                         break;
2507                 case SVM_EXITINTINFO_TYPE_EXEPT:
2508                         if (svm->vmcb->control.exit_info_2 &
2509                             (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
2510                                 has_error_code = true;
2511                                 error_code =
2512                                         (u32)svm->vmcb->control.exit_info_2;
2513                         }
2514                         kvm_clear_exception_queue(&svm->vcpu);
2515                         break;
2516                 case SVM_EXITINTINFO_TYPE_INTR:
2517                         kvm_clear_interrupt_queue(&svm->vcpu);
2518                         break;
2519                 default:
2520                         break;
2521                 }
2522         }
2523
2524         if (reason != TASK_SWITCH_GATE ||
2525             int_type == SVM_EXITINTINFO_TYPE_SOFT ||
2526             (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
2527              (int_vec == OF_VECTOR || int_vec == BP_VECTOR)))
2528                 skip_emulated_instruction(&svm->vcpu);
2529
2530         if (kvm_task_switch(&svm->vcpu, tss_selector, reason,
2531                                 has_error_code, error_code) == EMULATE_FAIL) {
2532                 svm->vcpu.run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
2533                 svm->vcpu.run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
2534                 svm->vcpu.run->internal.ndata = 0;
2535                 return 0;
2536         }
2537         return 1;
2538 }
2539
2540 static int cpuid_interception(struct vcpu_svm *svm)
2541 {
2542         svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
2543         kvm_emulate_cpuid(&svm->vcpu);
2544         return 1;
2545 }
2546
2547 static int iret_interception(struct vcpu_svm *svm)
2548 {
2549         ++svm->vcpu.stat.nmi_window_exits;
2550         svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_IRET);
2551         svm->vcpu.arch.hflags |= HF_IRET_MASK;
2552         return 1;
2553 }
2554
2555 static int invlpg_interception(struct vcpu_svm *svm)
2556 {
2557         return emulate_instruction(&svm->vcpu, 0, 0, 0) == EMULATE_DONE;
2558 }
2559
2560 static int emulate_on_interception(struct vcpu_svm *svm)
2561 {
2562         return emulate_instruction(&svm->vcpu, 0, 0, 0) == EMULATE_DONE;
2563 }
2564
2565 static int cr0_write_interception(struct vcpu_svm *svm)
2566 {
2567         struct kvm_vcpu *vcpu = &svm->vcpu;
2568         int r;
2569
2570         r = emulate_instruction(&svm->vcpu, 0, 0, 0);
2571
2572         if (svm->nested.vmexit_rip) {
2573                 kvm_register_write(vcpu, VCPU_REGS_RIP, svm->nested.vmexit_rip);
2574                 kvm_register_write(vcpu, VCPU_REGS_RSP, svm->nested.vmexit_rsp);
2575                 kvm_register_write(vcpu, VCPU_REGS_RAX, svm->nested.vmexit_rax);
2576                 svm->nested.vmexit_rip = 0;
2577         }
2578
2579         return r == EMULATE_DONE;
2580 }
2581
2582 static int cr8_write_interception(struct vcpu_svm *svm)
2583 {
2584         struct kvm_run *kvm_run = svm->vcpu.run;
2585
2586         u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
2587         /* instruction emulation calls kvm_set_cr8() */
2588         emulate_instruction(&svm->vcpu, 0, 0, 0);
2589         if (irqchip_in_kernel(svm->vcpu.kvm)) {
2590                 clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
2591                 return 1;
2592         }
2593         if (cr8_prev <= kvm_get_cr8(&svm->vcpu))
2594                 return 1;
2595         kvm_run->exit_reason = KVM_EXIT_SET_TPR;
2596         return 0;
2597 }
2598
2599 static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data)
2600 {
2601         struct vcpu_svm *svm = to_svm(vcpu);
2602
2603         switch (ecx) {
2604         case MSR_IA32_TSC: {
2605                 u64 tsc_offset;
2606
2607                 if (is_guest_mode(vcpu))
2608                         tsc_offset = svm->nested.hsave->control.tsc_offset;
2609                 else
2610                         tsc_offset = svm->vmcb->control.tsc_offset;
2611
2612                 *data = tsc_offset + native_read_tsc();
2613                 break;
2614         }
2615         case MSR_STAR:
2616                 *data = svm->vmcb->save.star;
2617                 break;
2618 #ifdef CONFIG_X86_64
2619         case MSR_LSTAR:
2620                 *data = svm->vmcb->save.lstar;
2621                 break;
2622         case MSR_CSTAR:
2623                 *data = svm->vmcb->save.cstar;
2624                 break;
2625         case MSR_KERNEL_GS_BASE:
2626                 *data = svm->vmcb->save.kernel_gs_base;
2627                 break;
2628         case MSR_SYSCALL_MASK:
2629                 *data = svm->vmcb->save.sfmask;
2630                 break;
2631 #endif
2632         case MSR_IA32_SYSENTER_CS:
2633                 *data = svm->vmcb->save.sysenter_cs;
2634                 break;
2635         case MSR_IA32_SYSENTER_EIP:
2636                 *data = svm->sysenter_eip;
2637                 break;
2638         case MSR_IA32_SYSENTER_ESP:
2639                 *data = svm->sysenter_esp;
2640                 break;
2641         /*
2642          * Nobody will change the following 5 values in the VMCB so we can
2643          * safely return them on rdmsr. They will always be 0 until LBRV is
2644          * implemented.
2645          */
2646         case MSR_IA32_DEBUGCTLMSR:
2647                 *data = svm->vmcb->save.dbgctl;
2648                 break;
2649         case MSR_IA32_LASTBRANCHFROMIP:
2650                 *data = svm->vmcb->save.br_from;
2651                 break;
2652         case MSR_IA32_LASTBRANCHTOIP:
2653                 *data = svm->vmcb->save.br_to;
2654                 break;
2655         case MSR_IA32_LASTINTFROMIP:
2656                 *data = svm->vmcb->save.last_excp_from;
2657                 break;
2658         case MSR_IA32_LASTINTTOIP:
2659                 *data = svm->vmcb->save.last_excp_to;
2660                 break;
2661         case MSR_VM_HSAVE_PA:
2662                 *data = svm->nested.hsave_msr;
2663                 break;
2664         case MSR_VM_CR:
2665                 *data = svm->nested.vm_cr_msr;
2666                 break;
2667         case MSR_IA32_UCODE_REV:
2668                 *data = 0x01000065;
2669                 break;
2670         default:
2671                 return kvm_get_msr_common(vcpu, ecx, data);
2672         }
2673         return 0;
2674 }
2675
2676 static int rdmsr_interception(struct vcpu_svm *svm)
2677 {
2678         u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX];
2679         u64 data;
2680
2681         if (svm_get_msr(&svm->vcpu, ecx, &data)) {
2682                 trace_kvm_msr_read_ex(ecx);
2683                 kvm_inject_gp(&svm->vcpu, 0);
2684         } else {
2685                 trace_kvm_msr_read(ecx, data);
2686
2687                 svm->vcpu.arch.regs[VCPU_REGS_RAX] = data & 0xffffffff;
2688                 svm->vcpu.arch.regs[VCPU_REGS_RDX] = data >> 32;
2689                 svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
2690                 skip_emulated_instruction(&svm->vcpu);
2691         }
2692         return 1;
2693 }
2694
2695 static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
2696 {
2697         struct vcpu_svm *svm = to_svm(vcpu);
2698         int svm_dis, chg_mask;
2699
2700         if (data & ~SVM_VM_CR_VALID_MASK)
2701                 return 1;
2702
2703         chg_mask = SVM_VM_CR_VALID_MASK;
2704
2705         if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
2706                 chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
2707
2708         svm->nested.vm_cr_msr &= ~chg_mask;
2709         svm->nested.vm_cr_msr |= (data & chg_mask);
2710
2711         svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
2712
2713         /* check for svm_disable while efer.svme is set */
2714         if (svm_dis && (vcpu->arch.efer & EFER_SVME))
2715                 return 1;
2716
2717         return 0;
2718 }
2719
2720 static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data)
2721 {
2722         struct vcpu_svm *svm = to_svm(vcpu);
2723
2724         switch (ecx) {
2725         case MSR_IA32_TSC:
2726                 kvm_write_tsc(vcpu, data);
2727                 break;
2728         case MSR_STAR:
2729                 svm->vmcb->save.star = data;
2730                 break;
2731 #ifdef CONFIG_X86_64
2732         case MSR_LSTAR:
2733                 svm->vmcb->save.lstar = data;
2734                 break;
2735         case MSR_CSTAR:
2736                 svm->vmcb->save.cstar = data;
2737                 break;
2738         case MSR_KERNEL_GS_BASE:
2739                 svm->vmcb->save.kernel_gs_base = data;
2740                 break;
2741         case MSR_SYSCALL_MASK:
2742                 svm->vmcb->save.sfmask = data;
2743                 break;
2744 #endif
2745         case MSR_IA32_SYSENTER_CS:
2746                 svm->vmcb->save.sysenter_cs = data;
2747                 break;
2748         case MSR_IA32_SYSENTER_EIP:
2749                 svm->sysenter_eip = data;
2750                 svm->vmcb->save.sysenter_eip = data;
2751                 break;
2752         case MSR_IA32_SYSENTER_ESP:
2753                 svm->sysenter_esp = data;
2754                 svm->vmcb->save.sysenter_esp = data;
2755                 break;
2756         case MSR_IA32_DEBUGCTLMSR:
2757                 if (!boot_cpu_has(X86_FEATURE_LBRV)) {
2758                         pr_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
2759                                         __func__, data);
2760                         break;
2761                 }
2762                 if (data & DEBUGCTL_RESERVED_BITS)
2763                         return 1;
2764
2765                 svm->vmcb->save.dbgctl = data;
2766                 if (data & (1ULL<<0))
2767                         svm_enable_lbrv(svm);
2768                 else
2769                         svm_disable_lbrv(svm);
2770                 break;
2771         case MSR_VM_HSAVE_PA:
2772                 svm->nested.hsave_msr = data;
2773                 break;
2774         case MSR_VM_CR:
2775                 return svm_set_vm_cr(vcpu, data);
2776         case MSR_VM_IGNNE:
2777                 pr_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
2778                 break;
2779         default:
2780                 return kvm_set_msr_common(vcpu, ecx, data);
2781         }
2782         return 0;
2783 }
2784
2785 static int wrmsr_interception(struct vcpu_svm *svm)
2786 {
2787         u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX];
2788         u64 data = (svm->vcpu.arch.regs[VCPU_REGS_RAX] & -1u)
2789                 | ((u64)(svm->vcpu.arch.regs[VCPU_REGS_RDX] & -1u) << 32);
2790
2791
2792         svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
2793         if (svm_set_msr(&svm->vcpu, ecx, data)) {
2794                 trace_kvm_msr_write_ex(ecx, data);
2795                 kvm_inject_gp(&svm->vcpu, 0);
2796         } else {
2797                 trace_kvm_msr_write(ecx, data);
2798                 skip_emulated_instruction(&svm->vcpu);
2799         }
2800         return 1;
2801 }
2802
2803 static int msr_interception(struct vcpu_svm *svm)
2804 {
2805         if (svm->vmcb->control.exit_info_1)
2806                 return wrmsr_interception(svm);
2807         else
2808                 return rdmsr_interception(svm);
2809 }
2810
2811 static int interrupt_window_interception(struct vcpu_svm *svm)
2812 {
2813         struct kvm_run *kvm_run = svm->vcpu.run;
2814
2815         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2816         svm_clear_vintr(svm);
2817         svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
2818         /*
2819          * If the user space waits to inject interrupts, exit as soon as
2820          * possible
2821          */
2822         if (!irqchip_in_kernel(svm->vcpu.kvm) &&
2823             kvm_run->request_interrupt_window &&
2824             !kvm_cpu_has_interrupt(&svm->vcpu)) {
2825                 ++svm->vcpu.stat.irq_window_exits;
2826                 kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
2827                 return 0;
2828         }
2829
2830         return 1;
2831 }
2832
2833 static int pause_interception(struct vcpu_svm *svm)
2834 {
2835         kvm_vcpu_on_spin(&(svm->vcpu));
2836         return 1;
2837 }
2838
2839 static int (*svm_exit_handlers[])(struct vcpu_svm *svm) = {
2840         [SVM_EXIT_READ_CR0]                     = emulate_on_interception,
2841         [SVM_EXIT_READ_CR3]                     = emulate_on_interception,
2842         [SVM_EXIT_READ_CR4]                     = emulate_on_interception,
2843         [SVM_EXIT_READ_CR8]                     = emulate_on_interception,
2844         [SVM_EXIT_CR0_SEL_WRITE]                = emulate_on_interception,
2845         [SVM_EXIT_WRITE_CR0]                    = cr0_write_interception,
2846         [SVM_EXIT_WRITE_CR3]                    = emulate_on_interception,
2847         [SVM_EXIT_WRITE_CR4]                    = emulate_on_interception,
2848         [SVM_EXIT_WRITE_CR8]                    = cr8_write_interception,
2849         [SVM_EXIT_READ_DR0]                     = emulate_on_interception,
2850         [SVM_EXIT_READ_DR1]                     = emulate_on_interception,
2851         [SVM_EXIT_READ_DR2]                     = emulate_on_interception,
2852         [SVM_EXIT_READ_DR3]                     = emulate_on_interception,
2853         [SVM_EXIT_READ_DR4]                     = emulate_on_interception,
2854         [SVM_EXIT_READ_DR5]                     = emulate_on_interception,
2855         [SVM_EXIT_READ_DR6]                     = emulate_on_interception,
2856         [SVM_EXIT_READ_DR7]                     = emulate_on_interception,
2857         [SVM_EXIT_WRITE_DR0]                    = emulate_on_interception,
2858         [SVM_EXIT_WRITE_DR1]                    = emulate_on_interception,
2859         [SVM_EXIT_WRITE_DR2]                    = emulate_on_interception,
2860         [SVM_EXIT_WRITE_DR3]                    = emulate_on_interception,
2861         [SVM_EXIT_WRITE_DR4]                    = emulate_on_interception,
2862         [SVM_EXIT_WRITE_DR5]                    = emulate_on_interception,
2863         [SVM_EXIT_WRITE_DR6]                    = emulate_on_interception,
2864         [SVM_EXIT_WRITE_DR7]                    = emulate_on_interception,
2865         [SVM_EXIT_EXCP_BASE + DB_VECTOR]        = db_interception,
2866         [SVM_EXIT_EXCP_BASE + BP_VECTOR]        = bp_interception,
2867         [SVM_EXIT_EXCP_BASE + UD_VECTOR]        = ud_interception,
2868         [SVM_EXIT_EXCP_BASE + PF_VECTOR]        = pf_interception,
2869         [SVM_EXIT_EXCP_BASE + NM_VECTOR]        = nm_interception,
2870         [SVM_EXIT_EXCP_BASE + MC_VECTOR]        = mc_interception,
2871         [SVM_EXIT_INTR]                         = intr_interception,
2872         [SVM_EXIT_NMI]                          = nmi_interception,
2873         [SVM_EXIT_SMI]                          = nop_on_interception,
2874         [SVM_EXIT_INIT]                         = nop_on_interception,
2875         [SVM_EXIT_VINTR]                        = interrupt_window_interception,
2876         [SVM_EXIT_CPUID]                        = cpuid_interception,
2877         [SVM_EXIT_IRET]                         = iret_interception,
2878         [SVM_EXIT_INVD]                         = emulate_on_interception,
2879         [SVM_EXIT_PAUSE]                        = pause_interception,
2880         [SVM_EXIT_HLT]                          = halt_interception,
2881         [SVM_EXIT_INVLPG]                       = invlpg_interception,
2882         [SVM_EXIT_INVLPGA]                      = invlpga_interception,
2883         [SVM_EXIT_IOIO]                         = io_interception,
2884         [SVM_EXIT_MSR]                          = msr_interception,
2885         [SVM_EXIT_TASK_SWITCH]                  = task_switch_interception,
2886         [SVM_EXIT_SHUTDOWN]                     = shutdown_interception,
2887         [SVM_EXIT_VMRUN]                        = vmrun_interception,
2888         [SVM_EXIT_VMMCALL]                      = vmmcall_interception,
2889         [SVM_EXIT_VMLOAD]                       = vmload_interception,
2890         [SVM_EXIT_VMSAVE]                       = vmsave_interception,
2891         [SVM_EXIT_STGI]                         = stgi_interception,
2892         [SVM_EXIT_CLGI]                         = clgi_interception,
2893         [SVM_EXIT_SKINIT]                       = skinit_interception,
2894         [SVM_EXIT_WBINVD]                       = emulate_on_interception,
2895         [SVM_EXIT_MONITOR]                      = invalid_op_interception,
2896         [SVM_EXIT_MWAIT]                        = invalid_op_interception,
2897         [SVM_EXIT_NPF]                          = pf_interception,
2898 };
2899
2900 void dump_vmcb(struct kvm_vcpu *vcpu)
2901 {
2902         struct vcpu_svm *svm = to_svm(vcpu);
2903         struct vmcb_control_area *control = &svm->vmcb->control;
2904         struct vmcb_save_area *save = &svm->vmcb->save;
2905
2906         pr_err("VMCB Control Area:\n");
2907         pr_err("cr_read:            %04x\n", control->intercept_cr & 0xffff);
2908         pr_err("cr_write:           %04x\n", control->intercept_cr >> 16);
2909         pr_err("dr_read:            %04x\n", control->intercept_dr_read);
2910         pr_err("dr_write:           %04x\n", control->intercept_dr_write);
2911         pr_err("exceptions:         %08x\n", control->intercept_exceptions);
2912         pr_err("intercepts:         %016llx\n", control->intercept);
2913         pr_err("pause filter count: %d\n", control->pause_filter_count);
2914         pr_err("iopm_base_pa:       %016llx\n", control->iopm_base_pa);
2915         pr_err("msrpm_base_pa:      %016llx\n", control->msrpm_base_pa);
2916         pr_err("tsc_offset:         %016llx\n", control->tsc_offset);
2917         pr_err("asid:               %d\n", control->asid);
2918         pr_err("tlb_ctl:            %d\n", control->tlb_ctl);
2919         pr_err("int_ctl:            %08x\n", control->int_ctl);
2920         pr_err("int_vector:         %08x\n", control->int_vector);
2921         pr_err("int_state:          %08x\n", control->int_state);
2922         pr_err("exit_code:          %08x\n", control->exit_code);
2923         pr_err("exit_info1:         %016llx\n", control->exit_info_1);
2924         pr_err("exit_info2:         %016llx\n", control->exit_info_2);
2925         pr_err("exit_int_info:      %08x\n", control->exit_int_info);
2926         pr_err("exit_int_info_err:  %08x\n", control->exit_int_info_err);
2927         pr_err("nested_ctl:         %lld\n", control->nested_ctl);
2928         pr_err("nested_cr3:         %016llx\n", control->nested_cr3);
2929         pr_err("event_inj:          %08x\n", control->event_inj);
2930         pr_err("event_inj_err:      %08x\n", control->event_inj_err);
2931         pr_err("lbr_ctl:            %lld\n", control->lbr_ctl);
2932         pr_err("next_rip:           %016llx\n", control->next_rip);
2933         pr_err("VMCB State Save Area:\n");
2934         pr_err("es:   s: %04x a: %04x l: %08x b: %016llx\n",
2935                 save->es.selector, save->es.attrib,
2936                 save->es.limit, save->es.base);
2937         pr_err("cs:   s: %04x a: %04x l: %08x b: %016llx\n",
2938                 save->cs.selector, save->cs.attrib,
2939                 save->cs.limit, save->cs.base);
2940         pr_err("ss:   s: %04x a: %04x l: %08x b: %016llx\n",
2941                 save->ss.selector, save->ss.attrib,
2942                 save->ss.limit, save->ss.base);
2943         pr_err("ds:   s: %04x a: %04x l: %08x b: %016llx\n",
2944                 save->ds.selector, save->ds.attrib,
2945                 save->ds.limit, save->ds.base);
2946         pr_err("fs:   s: %04x a: %04x l: %08x b: %016llx\n",
2947                 save->fs.selector, save->fs.attrib,
2948                 save->fs.limit, save->fs.base);
2949         pr_err("gs:   s: %04x a: %04x l: %08x b: %016llx\n",
2950                 save->gs.selector, save->gs.attrib,
2951                 save->gs.limit, save->gs.base);
2952         pr_err("gdtr: s: %04x a: %04x l: %08x b: %016llx\n",
2953                 save->gdtr.selector, save->gdtr.attrib,
2954                 save->gdtr.limit, save->gdtr.base);
2955         pr_err("ldtr: s: %04x a: %04x l: %08x b: %016llx\n",
2956                 save->ldtr.selector, save->ldtr.attrib,
2957                 save->ldtr.limit, save->ldtr.base);
2958         pr_err("idtr: s: %04x a: %04x l: %08x b: %016llx\n",
2959                 save->idtr.selector, save->idtr.attrib,
2960                 save->idtr.limit, save->idtr.base);
2961         pr_err("tr:   s: %04x a: %04x l: %08x b: %016llx\n",
2962                 save->tr.selector, save->tr.attrib,
2963                 save->tr.limit, save->tr.base);
2964         pr_err("cpl:            %d                efer:         %016llx\n",
2965                 save->cpl, save->efer);
2966         pr_err("cr0:            %016llx cr2:          %016llx\n",
2967                 save->cr0, save->cr2);
2968         pr_err("cr3:            %016llx cr4:          %016llx\n",
2969                 save->cr3, save->cr4);
2970         pr_err("dr6:            %016llx dr7:          %016llx\n",
2971                 save->dr6, save->dr7);
2972         pr_err("rip:            %016llx rflags:       %016llx\n",
2973                 save->rip, save->rflags);
2974         pr_err("rsp:            %016llx rax:          %016llx\n",
2975                 save->rsp, save->rax);
2976         pr_err("star:           %016llx lstar:        %016llx\n",
2977                 save->star, save->lstar);
2978         pr_err("cstar:          %016llx sfmask:       %016llx\n",
2979                 save->cstar, save->sfmask);
2980         pr_err("kernel_gs_base: %016llx sysenter_cs:  %016llx\n",
2981                 save->kernel_gs_base, save->sysenter_cs);
2982         pr_err("sysenter_esp:   %016llx sysenter_eip: %016llx\n",
2983                 save->sysenter_esp, save->sysenter_eip);
2984         pr_err("gpat:           %016llx dbgctl:       %016llx\n",
2985                 save->g_pat, save->dbgctl);
2986         pr_err("br_from:        %016llx br_to:        %016llx\n",
2987                 save->br_from, save->br_to);
2988         pr_err("excp_from:      %016llx excp_to:      %016llx\n",
2989                 save->last_excp_from, save->last_excp_to);
2990
2991 }
2992
2993 static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
2994 {
2995         struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
2996
2997         *info1 = control->exit_info_1;
2998         *info2 = control->exit_info_2;
2999 }
3000
3001 static int handle_exit(struct kvm_vcpu *vcpu)
3002 {
3003         struct vcpu_svm *svm = to_svm(vcpu);
3004         struct kvm_run *kvm_run = vcpu->run;
3005         u32 exit_code = svm->vmcb->control.exit_code;
3006
3007         trace_kvm_exit(exit_code, vcpu, KVM_ISA_SVM);
3008
3009         if (!is_cr_intercept(svm, INTERCEPT_CR0_WRITE))
3010                 vcpu->arch.cr0 = svm->vmcb->save.cr0;
3011         if (npt_enabled)
3012                 vcpu->arch.cr3 = svm->vmcb->save.cr3;
3013
3014         if (unlikely(svm->nested.exit_required)) {
3015                 nested_svm_vmexit(svm);
3016                 svm->nested.exit_required = false;
3017
3018                 return 1;
3019         }
3020
3021         if (is_guest_mode(vcpu)) {
3022                 int vmexit;
3023
3024                 trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code,
3025                                         svm->vmcb->control.exit_info_1,
3026                                         svm->vmcb->control.exit_info_2,
3027                                         svm->vmcb->control.exit_int_info,
3028                                         svm->vmcb->control.exit_int_info_err);
3029
3030                 vmexit = nested_svm_exit_special(svm);
3031
3032                 if (vmexit == NESTED_EXIT_CONTINUE)
3033                         vmexit = nested_svm_exit_handled(svm);
3034
3035                 if (vmexit == NESTED_EXIT_DONE)
3036                         return 1;
3037         }
3038
3039         svm_complete_interrupts(svm);
3040
3041         if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
3042                 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
3043                 kvm_run->fail_entry.hardware_entry_failure_reason
3044                         = svm->vmcb->control.exit_code;
3045                 pr_err("KVM: FAILED VMRUN WITH VMCB:\n");
3046                 dump_vmcb(vcpu);
3047                 return 0;
3048         }
3049
3050         if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
3051             exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
3052             exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH &&
3053             exit_code != SVM_EXIT_INTR && exit_code != SVM_EXIT_NMI)
3054                 printk(KERN_ERR "%s: unexpected exit_ini_info 0x%x "
3055                        "exit_code 0x%x\n",
3056                        __func__, svm->vmcb->control.exit_int_info,
3057                        exit_code);
3058
3059         if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
3060             || !svm_exit_handlers[exit_code]) {
3061                 kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
3062                 kvm_run->hw.hardware_exit_reason = exit_code;
3063                 return 0;
3064         }
3065
3066         return svm_exit_handlers[exit_code](svm);
3067 }
3068
3069 static void reload_tss(struct kvm_vcpu *vcpu)
3070 {
3071         int cpu = raw_smp_processor_id();
3072
3073         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
3074         sd->tss_desc->type = 9; /* available 32/64-bit TSS */
3075         load_TR_desc();
3076 }
3077
3078 static void pre_svm_run(struct vcpu_svm *svm)
3079 {
3080         int cpu = raw_smp_processor_id();
3081
3082         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
3083
3084         svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
3085         /* FIXME: handle wraparound of asid_generation */
3086         if (svm->asid_generation != sd->asid_generation)
3087                 new_asid(svm, sd);
3088 }
3089
3090 static void svm_inject_nmi(struct kvm_vcpu *vcpu)
3091 {
3092         struct vcpu_svm *svm = to_svm(vcpu);
3093
3094         svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
3095         vcpu->arch.hflags |= HF_NMI_MASK;
3096         svm->vmcb->control.intercept |= (1ULL << INTERCEPT_IRET);
3097         ++vcpu->stat.nmi_injections;
3098 }
3099
3100 static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
3101 {
3102         struct vmcb_control_area *control;
3103
3104         control = &svm->vmcb->control;
3105         control->int_vector = irq;
3106         control->int_ctl &= ~V_INTR_PRIO_MASK;
3107         control->int_ctl |= V_IRQ_MASK |
3108                 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
3109 }
3110
3111 static void svm_set_irq(struct kvm_vcpu *vcpu)
3112 {
3113         struct vcpu_svm *svm = to_svm(vcpu);
3114
3115         BUG_ON(!(gif_set(svm)));
3116
3117         trace_kvm_inj_virq(vcpu->arch.interrupt.nr);
3118         ++vcpu->stat.irq_injections;
3119
3120         svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
3121                 SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
3122 }
3123
3124 static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
3125 {
3126         struct vcpu_svm *svm = to_svm(vcpu);
3127
3128         if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK))
3129                 return;
3130
3131         if (irr == -1)
3132                 return;
3133
3134         if (tpr >= irr)
3135                 set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
3136 }
3137
3138 static int svm_nmi_allowed(struct kvm_vcpu *vcpu)
3139 {
3140         struct vcpu_svm *svm = to_svm(vcpu);
3141         struct vmcb *vmcb = svm->vmcb;
3142         int ret;
3143         ret = !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
3144               !(svm->vcpu.arch.hflags & HF_NMI_MASK);
3145         ret = ret && gif_set(svm) && nested_svm_nmi(svm);
3146
3147         return ret;
3148 }
3149
3150 static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
3151 {
3152         struct vcpu_svm *svm = to_svm(vcpu);
3153
3154         return !!(svm->vcpu.arch.hflags & HF_NMI_MASK);
3155 }
3156
3157 static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
3158 {
3159         struct vcpu_svm *svm = to_svm(vcpu);
3160
3161         if (masked) {
3162                 svm->vcpu.arch.hflags |= HF_NMI_MASK;
3163                 svm->vmcb->control.intercept |= (1ULL << INTERCEPT_IRET);
3164         } else {
3165                 svm->vcpu.arch.hflags &= ~HF_NMI_MASK;
3166                 svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_IRET);
3167         }
3168 }
3169
3170 static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
3171 {
3172         struct vcpu_svm *svm = to_svm(vcpu);
3173         struct vmcb *vmcb = svm->vmcb;
3174         int ret;
3175
3176         if (!gif_set(svm) ||
3177              (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK))
3178                 return 0;
3179
3180         ret = !!(vmcb->save.rflags & X86_EFLAGS_IF);
3181
3182         if (is_guest_mode(vcpu))
3183                 return ret && !(svm->vcpu.arch.hflags & HF_VINTR_MASK);
3184
3185         return ret;
3186 }
3187
3188 static void enable_irq_window(struct kvm_vcpu *vcpu)
3189 {
3190         struct vcpu_svm *svm = to_svm(vcpu);
3191
3192         /*
3193          * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
3194          * 1, because that's a separate STGI/VMRUN intercept.  The next time we
3195          * get that intercept, this function will be called again though and
3196          * we'll get the vintr intercept.
3197          */
3198         if (gif_set(svm) && nested_svm_intr(svm)) {
3199                 svm_set_vintr(svm);
3200                 svm_inject_irq(svm, 0x0);
3201         }
3202 }
3203
3204 static void enable_nmi_window(struct kvm_vcpu *vcpu)
3205 {
3206         struct vcpu_svm *svm = to_svm(vcpu);
3207
3208         if ((svm->vcpu.arch.hflags & (HF_NMI_MASK | HF_IRET_MASK))
3209             == HF_NMI_MASK)
3210                 return; /* IRET will cause a vm exit */
3211
3212         /*
3213          * Something prevents NMI from been injected. Single step over possible
3214          * problem (IRET or exception injection or interrupt shadow)
3215          */
3216         svm->nmi_singlestep = true;
3217         svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
3218         update_db_intercept(vcpu);
3219 }
3220
3221 static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
3222 {
3223         return 0;
3224 }
3225
3226 static void svm_flush_tlb(struct kvm_vcpu *vcpu)
3227 {
3228         force_new_asid(vcpu);
3229 }
3230
3231 static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
3232 {
3233 }
3234
3235 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
3236 {
3237         struct vcpu_svm *svm = to_svm(vcpu);
3238
3239         if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK))
3240                 return;
3241
3242         if (!is_cr_intercept(svm, INTERCEPT_CR8_WRITE)) {
3243                 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
3244                 kvm_set_cr8(vcpu, cr8);
3245         }
3246 }
3247
3248 static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
3249 {
3250         struct vcpu_svm *svm = to_svm(vcpu);
3251         u64 cr8;
3252
3253         if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK))
3254                 return;
3255
3256         cr8 = kvm_get_cr8(vcpu);
3257         svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
3258         svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
3259 }
3260
3261 static void svm_complete_interrupts(struct vcpu_svm *svm)
3262 {
3263         u8 vector;
3264         int type;
3265         u32 exitintinfo = svm->vmcb->control.exit_int_info;
3266         unsigned int3_injected = svm->int3_injected;
3267
3268         svm->int3_injected = 0;
3269
3270         if (svm->vcpu.arch.hflags & HF_IRET_MASK) {
3271                 svm->vcpu.arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK);
3272                 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3273         }
3274
3275         svm->vcpu.arch.nmi_injected = false;
3276         kvm_clear_exception_queue(&svm->vcpu);
3277         kvm_clear_interrupt_queue(&svm->vcpu);
3278
3279         if (!(exitintinfo & SVM_EXITINTINFO_VALID))
3280                 return;
3281
3282         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3283
3284         vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
3285         type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
3286
3287         switch (type) {
3288         case SVM_EXITINTINFO_TYPE_NMI:
3289                 svm->vcpu.arch.nmi_injected = true;
3290                 break;
3291         case SVM_EXITINTINFO_TYPE_EXEPT:
3292                 /*
3293                  * In case of software exceptions, do not reinject the vector,
3294                  * but re-execute the instruction instead. Rewind RIP first
3295                  * if we emulated INT3 before.
3296                  */
3297                 if (kvm_exception_is_soft(vector)) {
3298                         if (vector == BP_VECTOR && int3_injected &&
3299                             kvm_is_linear_rip(&svm->vcpu, svm->int3_rip))
3300                                 kvm_rip_write(&svm->vcpu,
3301                                               kvm_rip_read(&svm->vcpu) -
3302                                               int3_injected);
3303                         break;
3304                 }
3305                 if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
3306                         u32 err = svm->vmcb->control.exit_int_info_err;
3307                         kvm_requeue_exception_e(&svm->vcpu, vector, err);
3308
3309                 } else
3310                         kvm_requeue_exception(&svm->vcpu, vector);
3311                 break;
3312         case SVM_EXITINTINFO_TYPE_INTR:
3313                 kvm_queue_interrupt(&svm->vcpu, vector, false);
3314                 break;
3315         default:
3316                 break;
3317         }
3318 }
3319
3320 static void svm_cancel_injection(struct kvm_vcpu *vcpu)
3321 {
3322         struct vcpu_svm *svm = to_svm(vcpu);
3323         struct vmcb_control_area *control = &svm->vmcb->control;
3324
3325         control->exit_int_info = control->event_inj;
3326         control->exit_int_info_err = control->event_inj_err;
3327         control->event_inj = 0;
3328         svm_complete_interrupts(svm);
3329 }
3330
3331 #ifdef CONFIG_X86_64
3332 #define R "r"
3333 #else
3334 #define R "e"
3335 #endif
3336
3337 static void svm_vcpu_run(struct kvm_vcpu *vcpu)
3338 {
3339         struct vcpu_svm *svm = to_svm(vcpu);
3340
3341         svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
3342         svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
3343         svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
3344
3345         /*
3346          * A vmexit emulation is required before the vcpu can be executed
3347          * again.
3348          */
3349         if (unlikely(svm->nested.exit_required))
3350                 return;
3351
3352         pre_svm_run(svm);
3353
3354         sync_lapic_to_cr8(vcpu);
3355
3356         svm->vmcb->save.cr2 = vcpu->arch.cr2;
3357
3358         clgi();
3359
3360         local_irq_enable();
3361
3362         asm volatile (
3363                 "push %%"R"bp; \n\t"
3364                 "mov %c[rbx](%[svm]), %%"R"bx \n\t"
3365                 "mov %c[rcx](%[svm]), %%"R"cx \n\t"
3366                 "mov %c[rdx](%[svm]), %%"R"dx \n\t"
3367                 "mov %c[rsi](%[svm]), %%"R"si \n\t"
3368                 "mov %c[rdi](%[svm]), %%"R"di \n\t"
3369                 "mov %c[rbp](%[svm]), %%"R"bp \n\t"
3370 #ifdef CONFIG_X86_64
3371                 "mov %c[r8](%[svm]),  %%r8  \n\t"
3372                 "mov %c[r9](%[svm]),  %%r9  \n\t"
3373                 "mov %c[r10](%[svm]), %%r10 \n\t"
3374                 "mov %c[r11](%[svm]), %%r11 \n\t"
3375                 "mov %c[r12](%[svm]), %%r12 \n\t"
3376                 "mov %c[r13](%[svm]), %%r13 \n\t"
3377                 "mov %c[r14](%[svm]), %%r14 \n\t"
3378                 "mov %c[r15](%[svm]), %%r15 \n\t"
3379 #endif
3380
3381                 /* Enter guest mode */
3382                 "push %%"R"ax \n\t"
3383                 "mov %c[vmcb](%[svm]), %%"R"ax \n\t"
3384                 __ex(SVM_VMLOAD) "\n\t"
3385                 __ex(SVM_VMRUN) "\n\t"
3386                 __ex(SVM_VMSAVE) "\n\t"
3387                 "pop %%"R"ax \n\t"
3388
3389                 /* Save guest registers, load host registers */
3390                 "mov %%"R"bx, %c[rbx](%[svm]) \n\t"
3391                 "mov %%"R"cx, %c[rcx](%[svm]) \n\t"
3392                 "mov %%"R"dx, %c[rdx](%[svm]) \n\t"
3393                 "mov %%"R"si, %c[rsi](%[svm]) \n\t"
3394                 "mov %%"R"di, %c[rdi](%[svm]) \n\t"
3395                 "mov %%"R"bp, %c[rbp](%[svm]) \n\t"
3396 #ifdef CONFIG_X86_64
3397                 "mov %%r8,  %c[r8](%[svm]) \n\t"
3398                 "mov %%r9,  %c[r9](%[svm]) \n\t"
3399                 "mov %%r10, %c[r10](%[svm]) \n\t"
3400                 "mov %%r11, %c[r11](%[svm]) \n\t"
3401                 "mov %%r12, %c[r12](%[svm]) \n\t"
3402                 "mov %%r13, %c[r13](%[svm]) \n\t"
3403                 "mov %%r14, %c[r14](%[svm]) \n\t"
3404                 "mov %%r15, %c[r15](%[svm]) \n\t"
3405 #endif
3406                 "pop %%"R"bp"
3407                 :
3408                 : [svm]"a"(svm),
3409                   [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)),
3410                   [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])),
3411                   [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])),
3412                   [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])),
3413                   [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])),
3414                   [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])),
3415                   [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP]))
3416 #ifdef CONFIG_X86_64
3417                   , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])),
3418                   [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])),
3419                   [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])),
3420                   [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])),
3421                   [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])),
3422                   [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])),
3423                   [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])),
3424                   [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15]))
3425 #endif
3426                 : "cc", "memory"
3427                 , R"bx", R"cx", R"dx", R"si", R"di"
3428 #ifdef CONFIG_X86_64
3429                 , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15"
3430 #endif
3431                 );
3432
3433 #ifdef CONFIG_X86_64
3434         wrmsrl(MSR_GS_BASE, svm->host.gs_base);
3435 #else
3436         loadsegment(fs, svm->host.fs);
3437 #endif
3438
3439         reload_tss(vcpu);
3440
3441         local_irq_disable();
3442
3443         stgi();
3444
3445         vcpu->arch.cr2 = svm->vmcb->save.cr2;
3446         vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
3447         vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
3448         vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
3449
3450         sync_cr8_to_lapic(vcpu);
3451
3452         svm->next_rip = 0;
3453
3454         /* if exit due to PF check for async PF */
3455         if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
3456                 svm->apf_reason = kvm_read_and_reset_pf_reason();
3457
3458         if (npt_enabled) {
3459                 vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR);
3460                 vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR);
3461         }
3462
3463         /*
3464          * We need to handle MC intercepts here before the vcpu has a chance to
3465          * change the physical cpu
3466          */
3467         if (unlikely(svm->vmcb->control.exit_code ==
3468                      SVM_EXIT_EXCP_BASE + MC_VECTOR))
3469                 svm_handle_mce(svm);
3470 }
3471
3472 #undef R
3473
3474 static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
3475 {
3476         struct vcpu_svm *svm = to_svm(vcpu);
3477
3478         svm->vmcb->save.cr3 = root;
3479         force_new_asid(vcpu);
3480 }
3481
3482 static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)
3483 {
3484         struct vcpu_svm *svm = to_svm(vcpu);
3485
3486         svm->vmcb->control.nested_cr3 = root;
3487
3488         /* Also sync guest cr3 here in case we live migrate */
3489         svm->vmcb->save.cr3 = vcpu->arch.cr3;
3490
3491         force_new_asid(vcpu);
3492 }
3493
3494 static int is_disabled(void)
3495 {
3496         u64 vm_cr;
3497
3498         rdmsrl(MSR_VM_CR, vm_cr);
3499         if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
3500                 return 1;
3501
3502         return 0;
3503 }
3504
3505 static void
3506 svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
3507 {
3508         /*
3509          * Patch in the VMMCALL instruction:
3510          */
3511         hypercall[0] = 0x0f;
3512         hypercall[1] = 0x01;
3513         hypercall[2] = 0xd9;
3514 }
3515
3516 static void svm_check_processor_compat(void *rtn)
3517 {
3518         *(int *)rtn = 0;
3519 }
3520
3521 static bool svm_cpu_has_accelerated_tpr(void)
3522 {
3523         return false;
3524 }
3525
3526 static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
3527 {
3528         return 0;
3529 }
3530
3531 static void svm_cpuid_update(struct kvm_vcpu *vcpu)
3532 {
3533 }
3534
3535 static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
3536 {
3537         switch (func) {
3538         case 0x00000001:
3539                 /* Mask out xsave bit as long as it is not supported by SVM */
3540                 entry->ecx &= ~(bit(X86_FEATURE_XSAVE));
3541                 break;
3542         case 0x80000001:
3543                 if (nested)
3544                         entry->ecx |= (1 << 2); /* Set SVM bit */
3545                 break;
3546         case 0x8000000A:
3547                 entry->eax = 1; /* SVM revision 1 */
3548                 entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper
3549                                    ASID emulation to nested SVM */
3550                 entry->ecx = 0; /* Reserved */
3551                 entry->edx = 0; /* Per default do not support any
3552                                    additional features */
3553
3554                 /* Support next_rip if host supports it */
3555                 if (boot_cpu_has(X86_FEATURE_NRIPS))
3556                         entry->edx |= SVM_FEATURE_NRIP;
3557
3558                 /* Support NPT for the guest if enabled */
3559                 if (npt_enabled)
3560                         entry->edx |= SVM_FEATURE_NPT;
3561
3562                 break;
3563         }
3564 }
3565
3566 static const struct trace_print_flags svm_exit_reasons_str[] = {
3567         { SVM_EXIT_READ_CR0,                    "read_cr0" },
3568         { SVM_EXIT_READ_CR3,                    "read_cr3" },
3569         { SVM_EXIT_READ_CR4,                    "read_cr4" },
3570         { SVM_EXIT_READ_CR8,                    "read_cr8" },
3571         { SVM_EXIT_WRITE_CR0,                   "write_cr0" },
3572         { SVM_EXIT_WRITE_CR3,                   "write_cr3" },
3573         { SVM_EXIT_WRITE_CR4,                   "write_cr4" },
3574         { SVM_EXIT_WRITE_CR8,                   "write_cr8" },
3575         { SVM_EXIT_READ_DR0,                    "read_dr0" },
3576         { SVM_EXIT_READ_DR1,                    "read_dr1" },
3577         { SVM_EXIT_READ_DR2,                    "read_dr2" },
3578         { SVM_EXIT_READ_DR3,                    "read_dr3" },
3579         { SVM_EXIT_WRITE_DR0,                   "write_dr0" },
3580         { SVM_EXIT_WRITE_DR1,                   "write_dr1" },
3581         { SVM_EXIT_WRITE_DR2,                   "write_dr2" },
3582         { SVM_EXIT_WRITE_DR3,                   "write_dr3" },
3583         { SVM_EXIT_WRITE_DR5,                   "write_dr5" },
3584         { SVM_EXIT_WRITE_DR7,                   "write_dr7" },
3585         { SVM_EXIT_EXCP_BASE + DB_VECTOR,       "DB excp" },
3586         { SVM_EXIT_EXCP_BASE + BP_VECTOR,       "BP excp" },
3587         { SVM_EXIT_EXCP_BASE + UD_VECTOR,       "UD excp" },
3588         { SVM_EXIT_EXCP_BASE + PF_VECTOR,       "PF excp" },
3589         { SVM_EXIT_EXCP_BASE + NM_VECTOR,       "NM excp" },
3590         { SVM_EXIT_EXCP_BASE + MC_VECTOR,       "MC excp" },
3591         { SVM_EXIT_INTR,                        "interrupt" },
3592         { SVM_EXIT_NMI,                         "nmi" },
3593         { SVM_EXIT_SMI,                         "smi" },
3594         { SVM_EXIT_INIT,                        "init" },
3595         { SVM_EXIT_VINTR,                       "vintr" },
3596         { SVM_EXIT_CPUID,                       "cpuid" },
3597         { SVM_EXIT_INVD,                        "invd" },
3598         { SVM_EXIT_HLT,                         "hlt" },
3599         { SVM_EXIT_INVLPG,                      "invlpg" },
3600         { SVM_EXIT_INVLPGA,                     "invlpga" },
3601         { SVM_EXIT_IOIO,                        "io" },
3602         { SVM_EXIT_MSR,                         "msr" },
3603         { SVM_EXIT_TASK_SWITCH,                 "task_switch" },
3604         { SVM_EXIT_SHUTDOWN,                    "shutdown" },
3605         { SVM_EXIT_VMRUN,                       "vmrun" },
3606         { SVM_EXIT_VMMCALL,                     "hypercall" },
3607         { SVM_EXIT_VMLOAD,                      "vmload" },
3608         { SVM_EXIT_VMSAVE,                      "vmsave" },
3609         { SVM_EXIT_STGI,                        "stgi" },
3610         { SVM_EXIT_CLGI,                        "clgi" },
3611         { SVM_EXIT_SKINIT,                      "skinit" },
3612         { SVM_EXIT_WBINVD,                      "wbinvd" },
3613         { SVM_EXIT_MONITOR,                     "monitor" },
3614         { SVM_EXIT_MWAIT,                       "mwait" },
3615         { SVM_EXIT_NPF,                         "npf" },
3616         { -1, NULL }
3617 };
3618
3619 static int svm_get_lpage_level(void)
3620 {
3621         return PT_PDPE_LEVEL;
3622 }
3623
3624 static bool svm_rdtscp_supported(void)
3625 {
3626         return false;
3627 }
3628
3629 static bool svm_has_wbinvd_exit(void)
3630 {
3631         return true;
3632 }
3633
3634 static void svm_fpu_deactivate(struct kvm_vcpu *vcpu)
3635 {
3636         struct vcpu_svm *svm = to_svm(vcpu);
3637
3638         svm->vmcb->control.intercept_exceptions |= 1 << NM_VECTOR;
3639         if (is_guest_mode(vcpu))
3640                 svm->nested.hsave->control.intercept_exceptions |= 1 << NM_VECTOR;
3641         update_cr0_intercept(svm);
3642 }
3643
3644 static struct kvm_x86_ops svm_x86_ops = {
3645         .cpu_has_kvm_support = has_svm,
3646         .disabled_by_bios = is_disabled,
3647         .hardware_setup = svm_hardware_setup,
3648         .hardware_unsetup = svm_hardware_unsetup,
3649         .check_processor_compatibility = svm_check_processor_compat,
3650         .hardware_enable = svm_hardware_enable,
3651         .hardware_disable = svm_hardware_disable,
3652         .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
3653
3654         .vcpu_create = svm_create_vcpu,
3655         .vcpu_free = svm_free_vcpu,
3656         .vcpu_reset = svm_vcpu_reset,
3657
3658         .prepare_guest_switch = svm_prepare_guest_switch,
3659         .vcpu_load = svm_vcpu_load,
3660         .vcpu_put = svm_vcpu_put,
3661
3662         .set_guest_debug = svm_guest_debug,
3663         .get_msr = svm_get_msr,
3664         .set_msr = svm_set_msr,
3665         .get_segment_base = svm_get_segment_base,
3666         .get_segment = svm_get_segment,
3667         .set_segment = svm_set_segment,
3668         .get_cpl = svm_get_cpl,
3669         .get_cs_db_l_bits = kvm_get_cs_db_l_bits,
3670         .decache_cr0_guest_bits = svm_decache_cr0_guest_bits,
3671         .decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
3672         .set_cr0 = svm_set_cr0,
3673         .set_cr3 = svm_set_cr3,
3674         .set_cr4 = svm_set_cr4,
3675         .set_efer = svm_set_efer,
3676         .get_idt = svm_get_idt,
3677         .set_idt = svm_set_idt,
3678         .get_gdt = svm_get_gdt,
3679         .set_gdt = svm_set_gdt,
3680         .set_dr7 = svm_set_dr7,
3681         .cache_reg = svm_cache_reg,
3682         .get_rflags = svm_get_rflags,
3683         .set_rflags = svm_set_rflags,
3684         .fpu_activate = svm_fpu_activate,
3685         .fpu_deactivate = svm_fpu_deactivate,
3686
3687         .tlb_flush = svm_flush_tlb,
3688
3689         .run = svm_vcpu_run,
3690         .handle_exit = handle_exit,
3691         .skip_emulated_instruction = skip_emulated_instruction,
3692         .set_interrupt_shadow = svm_set_interrupt_shadow,
3693         .get_interrupt_shadow = svm_get_interrupt_shadow,
3694         .patch_hypercall = svm_patch_hypercall,
3695         .set_irq = svm_set_irq,
3696         .set_nmi = svm_inject_nmi,
3697         .queue_exception = svm_queue_exception,
3698         .cancel_injection = svm_cancel_injection,
3699         .interrupt_allowed = svm_interrupt_allowed,
3700         .nmi_allowed = svm_nmi_allowed,
3701         .get_nmi_mask = svm_get_nmi_mask,
3702         .set_nmi_mask = svm_set_nmi_mask,
3703         .enable_nmi_window = enable_nmi_window,
3704         .enable_irq_window = enable_irq_window,
3705         .update_cr8_intercept = update_cr8_intercept,
3706
3707         .set_tss_addr = svm_set_tss_addr,
3708         .get_tdp_level = get_npt_level,
3709         .get_mt_mask = svm_get_mt_mask,
3710
3711         .get_exit_info = svm_get_exit_info,
3712         .exit_reasons_str = svm_exit_reasons_str,
3713
3714         .get_lpage_level = svm_get_lpage_level,
3715
3716         .cpuid_update = svm_cpuid_update,
3717
3718         .rdtscp_supported = svm_rdtscp_supported,
3719
3720         .set_supported_cpuid = svm_set_supported_cpuid,
3721
3722         .has_wbinvd_exit = svm_has_wbinvd_exit,
3723
3724         .write_tsc_offset = svm_write_tsc_offset,
3725         .adjust_tsc_offset = svm_adjust_tsc_offset,
3726
3727         .set_tdp_cr3 = set_tdp_cr3,
3728 };
3729
3730 static int __init svm_init(void)
3731 {
3732         return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm),
3733                         __alignof__(struct vcpu_svm), THIS_MODULE);
3734 }
3735
3736 static void __exit svm_exit(void)
3737 {
3738         kvm_exit();
3739 }
3740
3741 module_init(svm_init)
3742 module_exit(svm_exit)