arch/powerpc/kvm/book3s_pr.c

   1 /*
   2  * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved.
   3  *
   4  * Authors:
   5  *    Alexander Graf <agraf@suse.de>
   6  *    Kevin Wolf <mail@kevin-wolf.de>
   7  *    Paul Mackerras <paulus@samba.org>
   8  *
   9  * Description:
  10  * Functions relating to running KVM on Book 3S processors where
  11  * we don't have access to hypervisor mode, and we run the guest
  12  * in problem state (user mode).
  13  *
  14  * This file is derived from arch/powerpc/kvm/44x.c,
  15  * by Hollis Blanchard <hollisb@us.ibm.com>.
  16  *
  17  * This program is free software; you can redistribute it and/or modify
  18  * it under the terms of the GNU General Public License, version 2, as
  19  * published by the Free Software Foundation.
  20  */
  21
  22 #include <linux/kvm_host.h>
  23 #include <linux/err.h>
  24 #include <linux/slab.h>
  25
  26 #include <asm/reg.h>
  27 #include <asm/cputable.h>
  28 #include <asm/cacheflush.h>
  29 #include <asm/tlbflush.h>
  30 #include <asm/uaccess.h>
  31 #include <asm/io.h>
  32 #include <asm/kvm_ppc.h>
  33 #include <asm/kvm_book3s.h>
  34 #include <asm/mmu_context.h>
  35 #include <linux/gfp.h>
  36 #include <linux/sched.h>
  37 #include <linux/vmalloc.h>
  38 #include <linux/highmem.h>
  39
  40 #include "trace.h"
  41
  42 /* #define EXIT_DEBUG */
  43 /* #define DEBUG_EXT */
  44
  45 static int kvmppc_handle_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr,
  46                              ulong msr);
  47
  48 /* Some compatibility defines */
  49 #ifdef CONFIG_PPC_BOOK3S_32
  50 #define MSR_USER32 MSR_USER
  51 #define MSR_USER64 MSR_USER
  52 #define HW_PAGE_SIZE PAGE_SIZE
  53 #endif
  54
  55 void kvmppc_core_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
  56 {
  57 #ifdef CONFIG_PPC_BOOK3S_64
  58         memcpy(to_svcpu(vcpu)->slb, to_book3s(vcpu)->slb_shadow, sizeof(to_svcpu(vcpu)->slb));
  59         memcpy(&get_paca()->shadow_vcpu, to_book3s(vcpu)->shadow_vcpu,
  60                sizeof(get_paca()->shadow_vcpu));
  61         to_svcpu(vcpu)->slb_max = to_book3s(vcpu)->slb_shadow_max;
  62 #endif
  63
  64 #ifdef CONFIG_PPC_BOOK3S_32
  65         current->thread.kvm_shadow_vcpu = to_book3s(vcpu)->shadow_vcpu;
  66 #endif
  67 }
  68
  69 void kvmppc_core_vcpu_put(struct kvm_vcpu *vcpu)
  70 {
  71 #ifdef CONFIG_PPC_BOOK3S_64
  72         memcpy(to_book3s(vcpu)->slb_shadow, to_svcpu(vcpu)->slb, sizeof(to_svcpu(vcpu)->slb));
  73         memcpy(to_book3s(vcpu)->shadow_vcpu, &get_paca()->shadow_vcpu,
  74                sizeof(get_paca()->shadow_vcpu));
  75         to_book3s(vcpu)->slb_shadow_max = to_svcpu(vcpu)->slb_max;
  76 #endif
  77
  78         kvmppc_giveup_ext(vcpu, MSR_FP);
  79         kvmppc_giveup_ext(vcpu, MSR_VEC);
  80         kvmppc_giveup_ext(vcpu, MSR_VSX);
  81 }
  82
  83 static void kvmppc_recalc_shadow_msr(struct kvm_vcpu *vcpu)
  84 {
  85         ulong smsr = vcpu->arch.shared->msr;
  86
  87         /* Guest MSR values */
  88         smsr &= MSR_FE0 | MSR_FE1 | MSR_SF | MSR_SE | MSR_BE | MSR_DE;
  89         /* Process MSR values */
  90         smsr |= MSR_ME | MSR_RI | MSR_IR | MSR_DR | MSR_PR | MSR_EE;
  91         /* External providers the guest reserved */
  92         smsr |= (vcpu->arch.shared->msr & vcpu->arch.guest_owned_ext);
  93         /* 64-bit Process MSR values */
  94 #ifdef CONFIG_PPC_BOOK3S_64
  95         smsr |= MSR_ISF | MSR_HV;
  96 #endif
  97         vcpu->arch.shadow_msr = smsr;
  98 }
  99
 100 void kvmppc_set_msr(struct kvm_vcpu *vcpu, u64 msr)
 101 {
 102         ulong old_msr = vcpu->arch.shared->msr;
 103
 104 #ifdef EXIT_DEBUG
 105         printk(KERN_INFO "KVM: Set MSR to 0x%llx\n", msr);
 106 #endif
 107
 108         msr &= to_book3s(vcpu)->msr_mask;
 109         vcpu->arch.shared->msr = msr;
 110         kvmppc_recalc_shadow_msr(vcpu);
 111
 112         if (msr & MSR_POW) {
 113                 if (!vcpu->arch.pending_exceptions) {
 114                         kvm_vcpu_block(vcpu);
 115                         vcpu->stat.halt_wakeup++;
 116
 117                         /* Unset POW bit after we woke up */
 118                         msr &= ~MSR_POW;
 119                         vcpu->arch.shared->msr = msr;
 120                 }
 121         }
 122
 123         if ((vcpu->arch.shared->msr & (MSR_PR|MSR_IR|MSR_DR)) !=
 124                    (old_msr & (MSR_PR|MSR_IR|MSR_DR))) {
 125                 kvmppc_mmu_flush_segments(vcpu);
 126                 kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu));
 127
 128                 /* Preload magic page segment when in kernel mode */
 129                 if (!(msr & MSR_PR) && vcpu->arch.magic_page_pa) {
 130                         struct kvm_vcpu_arch *a = &vcpu->arch;
 131
 132                         if (msr & MSR_DR)
 133                                 kvmppc_mmu_map_segment(vcpu, a->magic_page_ea);
 134                         else
 135                                 kvmppc_mmu_map_segment(vcpu, a->magic_page_pa);
 136                 }
 137         }
 138
 139         /* Preload FPU if it's enabled */
 140         if (vcpu->arch.shared->msr & MSR_FP)
 141                 kvmppc_handle_ext(vcpu, BOOK3S_INTERRUPT_FP_UNAVAIL, MSR_FP);
 142 }
 143
 144 void kvmppc_set_pvr(struct kvm_vcpu *vcpu, u32 pvr)
 145 {
 146         u32 host_pvr;
 147
 148         vcpu->arch.hflags &= ~BOOK3S_HFLAG_SLB;
 149         vcpu->arch.pvr = pvr;
 150 #ifdef CONFIG_PPC_BOOK3S_64
 151         if ((pvr >= 0x330000) && (pvr < 0x70330000)) {
 152                 kvmppc_mmu_book3s_64_init(vcpu);
 153                 to_book3s(vcpu)->hior = 0xfff00000;
 154                 to_book3s(vcpu)->msr_mask = 0xffffffffffffffffULL;
 155         } else
 156 #endif
 157         {
 158                 kvmppc_mmu_book3s_32_init(vcpu);
 159                 to_book3s(vcpu)->hior = 0;
 160                 to_book3s(vcpu)->msr_mask = 0xffffffffULL;
 161         }
 162
 163         /* If we are in hypervisor level on 970, we can tell the CPU to
 164          * treat DCBZ as 32 bytes store */
 165         vcpu->arch.hflags &= ~BOOK3S_HFLAG_DCBZ32;
 166         if (vcpu->arch.mmu.is_dcbz32(vcpu) && (mfmsr() & MSR_HV) &&
 167             !strcmp(cur_cpu_spec->platform, "ppc970"))
 168                 vcpu->arch.hflags |= BOOK3S_HFLAG_DCBZ32;
 169
 170         /* Cell performs badly if MSR_FEx are set. So let's hope nobody
 171            really needs them in a VM on Cell and force disable them. */
 172         if (!strcmp(cur_cpu_spec->platform, "ppc-cell-be"))
 173                 to_book3s(vcpu)->msr_mask &= ~(MSR_FE0 | MSR_FE1);
 174
 175 #ifdef CONFIG_PPC_BOOK3S_32
 176         /* 32 bit Book3S always has 32 byte dcbz */
 177         vcpu->arch.hflags |= BOOK3S_HFLAG_DCBZ32;
 178 #endif
 179
 180         /* On some CPUs we can execute paired single operations natively */
 181         asm ( "mfpvr %0" : "=r"(host_pvr));
 182         switch (host_pvr) {
 183         case 0x00080200:        /* lonestar 2.0 */
 184         case 0x00088202:        /* lonestar 2.2 */
 185         case 0x70000100:        /* gekko 1.0 */
 186         case 0x00080100:        /* gekko 2.0 */
 187         case 0x00083203:        /* gekko 2.3a */
 188         case 0x00083213:        /* gekko 2.3b */
 189         case 0x00083204:        /* gekko 2.4 */
 190         case 0x00083214:        /* gekko 2.4e (8SE) - retail HW2 */
 191         case 0x00087200:        /* broadway */
 192                 vcpu->arch.hflags |= BOOK3S_HFLAG_NATIVE_PS;
 193                 /* Enable HID2.PSE - in case we need it later */
 194                 mtspr(SPRN_HID2_GEKKO, mfspr(SPRN_HID2_GEKKO) | (1 << 29));
 195         }
 196 }
 197
 198 /* Book3s_32 CPUs always have 32 bytes cache line size, which Linux assumes. To
 199  * make Book3s_32 Linux work on Book3s_64, we have to make sure we trap dcbz to
 200  * emulate 32 bytes dcbz length.
 201  *
 202  * The Book3s_64 inventors also realized this case and implemented a special bit
 203  * in the HID5 register, which is a hypervisor ressource. Thus we can't use it.
 204  *
 205  * My approach here is to patch the dcbz instruction on executing pages.
 206  */
 207 static void kvmppc_patch_dcbz(struct kvm_vcpu *vcpu, struct kvmppc_pte *pte)
 208 {
 209         struct page *hpage;
 210         u64 hpage_offset;
 211         u32 *page;
 212         int i;
 213
 214         hpage = gfn_to_page(vcpu->kvm, pte->raddr >> PAGE_SHIFT);
 215         if (is_error_page(hpage)) {
 216                 kvm_release_page_clean(hpage);
 217                 return;
 218         }
 219
 220         hpage_offset = pte->raddr & ~PAGE_MASK;
 221         hpage_offset &= ~0xFFFULL;
 222         hpage_offset /= 4;
 223
 224         get_page(hpage);
 225         page = kmap_atomic(hpage, KM_USER0);
 226
 227         /* patch dcbz into reserved instruction, so we trap */
 228         for (i=hpage_offset; i < hpage_offset + (HW_PAGE_SIZE / 4); i++)
 229                 if ((page[i] & 0xff0007ff) == INS_DCBZ)
 230                         page[i] &= 0xfffffff7;
 231
 232         kunmap_atomic(page, KM_USER0);
 233         put_page(hpage);
 234 }
 235
 236 static int kvmppc_visible_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
 237 {
 238         ulong mp_pa = vcpu->arch.magic_page_pa;
 239
 240         if (unlikely(mp_pa) &&
 241             unlikely((mp_pa & KVM_PAM) >> PAGE_SHIFT == gfn)) {
 242                 return 1;
 243         }
 244
 245         return kvm_is_visible_gfn(vcpu->kvm, gfn);
 246 }
 247
 248 int kvmppc_handle_pagefault(struct kvm_run *run, struct kvm_vcpu *vcpu,
 249                             ulong eaddr, int vec)
 250 {
 251         bool data = (vec == BOOK3S_INTERRUPT_DATA_STORAGE);
 252         int r = RESUME_GUEST;
 253         int relocated;
 254         int page_found = 0;
 255         struct kvmppc_pte pte;
 256         bool is_mmio = false;
 257         bool dr = (vcpu->arch.shared->msr & MSR_DR) ? true : false;
 258         bool ir = (vcpu->arch.shared->msr & MSR_IR) ? true : false;
 259         u64 vsid;
 260
 261         relocated = data ? dr : ir;
 262
 263         /* Resolve real address if translation turned on */
 264         if (relocated) {
 265                 page_found = vcpu->arch.mmu.xlate(vcpu, eaddr, &pte, data);
 266         } else {
 267                 pte.may_execute = true;
 268                 pte.may_read = true;
 269                 pte.may_write = true;
 270                 pte.raddr = eaddr & KVM_PAM;
 271                 pte.eaddr = eaddr;
 272                 pte.vpage = eaddr >> 12;
 273         }
 274
 275         switch (vcpu->arch.shared->msr & (MSR_DR|MSR_IR)) {
 276         case 0:
 277                 pte.vpage |= ((u64)VSID_REAL << (SID_SHIFT - 12));
 278                 break;
 279         case MSR_DR:
 280         case MSR_IR:
 281                 vcpu->arch.mmu.esid_to_vsid(vcpu, eaddr >> SID_SHIFT, &vsid);
 282
 283                 if ((vcpu->arch.shared->msr & (MSR_DR|MSR_IR)) == MSR_DR)
 284                         pte.vpage |= ((u64)VSID_REAL_DR << (SID_SHIFT - 12));
 285                 else
 286                         pte.vpage |= ((u64)VSID_REAL_IR << (SID_SHIFT - 12));
 287                 pte.vpage |= vsid;
 288
 289                 if (vsid == -1)
 290                         page_found = -EINVAL;
 291                 break;
 292         }
 293
 294         if (vcpu->arch.mmu.is_dcbz32(vcpu) &&
 295            (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32))) {
 296                 /*
 297                  * If we do the dcbz hack, we have to NX on every execution,
 298                  * so we can patch the executing code. This renders our guest
 299                  * NX-less.
 300                  */
 301                 pte.may_execute = !data;
 302         }
 303
 304         if (page_found == -ENOENT) {
 305                 /* Page not found in guest PTE entries */
 306                 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu);
 307                 vcpu->arch.shared->dsisr = to_svcpu(vcpu)->fault_dsisr;
 308                 vcpu->arch.shared->msr |=
 309                         (to_svcpu(vcpu)->shadow_srr1 & 0x00000000f8000000ULL);
 310                 kvmppc_book3s_queue_irqprio(vcpu, vec);
 311         } else if (page_found == -EPERM) {
 312                 /* Storage protection */
 313                 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu);
 314                 vcpu->arch.shared->dsisr =
 315                         to_svcpu(vcpu)->fault_dsisr & ~DSISR_NOHPTE;
 316                 vcpu->arch.shared->dsisr |= DSISR_PROTFAULT;
 317                 vcpu->arch.shared->msr |=
 318                         (to_svcpu(vcpu)->shadow_srr1 & 0x00000000f8000000ULL);
 319                 kvmppc_book3s_queue_irqprio(vcpu, vec);
 320         } else if (page_found == -EINVAL) {
 321                 /* Page not found in guest SLB */
 322                 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu);
 323                 kvmppc_book3s_queue_irqprio(vcpu, vec + 0x80);
 324         } else if (!is_mmio &&
 325                    kvmppc_visible_gfn(vcpu, pte.raddr >> PAGE_SHIFT)) {
 326                 /* The guest's PTE is not mapped yet. Map on the host */
 327                 kvmppc_mmu_map_page(vcpu, &pte);
 328                 if (data)
 329                         vcpu->stat.sp_storage++;
 330                 else if (vcpu->arch.mmu.is_dcbz32(vcpu) &&
 331                         (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32)))
 332                         kvmppc_patch_dcbz(vcpu, &pte);
 333         } else {
 334                 /* MMIO */
 335                 vcpu->stat.mmio_exits++;
 336                 vcpu->arch.paddr_accessed = pte.raddr;
 337                 r = kvmppc_emulate_mmio(run, vcpu);
 338                 if ( r == RESUME_HOST_NV )
 339                         r = RESUME_HOST;
 340         }
 341
 342         return r;
 343 }
 344
 345 static inline int get_fpr_index(int i)
 346 {
 347 #ifdef CONFIG_VSX
 348         i *= 2;
 349 #endif
 350         return i;
 351 }
 352
 353 /* Give up external provider (FPU, Altivec, VSX) */
 354 void kvmppc_giveup_ext(struct kvm_vcpu *vcpu, ulong msr)
 355 {
 356         struct thread_struct *t = &current->thread;
 357         u64 *vcpu_fpr = vcpu->arch.fpr;
 358 #ifdef CONFIG_VSX
 359         u64 *vcpu_vsx = vcpu->arch.vsr;
 360 #endif
 361         u64 *thread_fpr = (u64*)t->fpr;
 362         int i;
 363
 364         if (!(vcpu->arch.guest_owned_ext & msr))
 365                 return;
 366
 367 #ifdef DEBUG_EXT
 368         printk(KERN_INFO "Giving up ext 0x%lx\n", msr);
 369 #endif
 370
 371         switch (msr) {
 372         case MSR_FP:
 373                 giveup_fpu(current);
 374                 for (i = 0; i < ARRAY_SIZE(vcpu->arch.fpr); i++)
 375                         vcpu_fpr[i] = thread_fpr[get_fpr_index(i)];
 376
 377                 vcpu->arch.fpscr = t->fpscr.val;
 378                 break;
 379         case MSR_VEC:
 380 #ifdef CONFIG_ALTIVEC
 381                 giveup_altivec(current);
 382                 memcpy(vcpu->arch.vr, t->vr, sizeof(vcpu->arch.vr));
 383                 vcpu->arch.vscr = t->vscr;
 384 #endif
 385                 break;
 386         case MSR_VSX:
 387 #ifdef CONFIG_VSX
 388                 __giveup_vsx(current);
 389                 for (i = 0; i < ARRAY_SIZE(vcpu->arch.vsr); i++)
 390                         vcpu_vsx[i] = thread_fpr[get_fpr_index(i) + 1];
 391 #endif
 392                 break;
 393         default:
 394                 BUG();
 395         }
 396
 397         vcpu->arch.guest_owned_ext &= ~msr;
 398         current->thread.regs->msr &= ~msr;
 399         kvmppc_recalc_shadow_msr(vcpu);
 400 }
 401
 402 static int kvmppc_read_inst(struct kvm_vcpu *vcpu)
 403 {
 404         ulong srr0 = kvmppc_get_pc(vcpu);
 405         u32 last_inst = kvmppc_get_last_inst(vcpu);
 406         int ret;
 407
 408         ret = kvmppc_ld(vcpu, &srr0, sizeof(u32), &last_inst, false);
 409         if (ret == -ENOENT) {
 410                 ulong msr = vcpu->arch.shared->msr;
 411
 412                 msr = kvmppc_set_field(msr, 33, 33, 1);
 413                 msr = kvmppc_set_field(msr, 34, 36, 0);
 414                 vcpu->arch.shared->msr = kvmppc_set_field(msr, 42, 47, 0);
 415                 kvmppc_book3s_queue_irqprio(vcpu, BOOK3S_INTERRUPT_INST_STORAGE);
 416                 return EMULATE_AGAIN;
 417         }
 418
 419         return EMULATE_DONE;
 420 }
 421
 422 static int kvmppc_check_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr)
 423 {
 424
 425         /* Need to do paired single emulation? */
 426         if (!(vcpu->arch.hflags & BOOK3S_HFLAG_PAIRED_SINGLE))
 427                 return EMULATE_DONE;
 428
 429         /* Read out the instruction */
 430         if (kvmppc_read_inst(vcpu) == EMULATE_DONE)
 431                 /* Need to emulate */
 432                 return EMULATE_FAIL;
 433
 434         return EMULATE_AGAIN;
 435 }
 436
 437 /* Handle external providers (FPU, Altivec, VSX) */
 438 static int kvmppc_handle_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr,
 439                              ulong msr)
 440 {
 441         struct thread_struct *t = &current->thread;
 442         u64 *vcpu_fpr = vcpu->arch.fpr;
 443 #ifdef CONFIG_VSX
 444         u64 *vcpu_vsx = vcpu->arch.vsr;
 445 #endif
 446         u64 *thread_fpr = (u64*)t->fpr;
 447         int i;
 448
 449         /* When we have paired singles, we emulate in software */
 450         if (vcpu->arch.hflags & BOOK3S_HFLAG_PAIRED_SINGLE)
 451                 return RESUME_GUEST;
 452
 453         if (!(vcpu->arch.shared->msr & msr)) {
 454                 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
 455                 return RESUME_GUEST;
 456         }
 457
 458         /* We already own the ext */
 459         if (vcpu->arch.guest_owned_ext & msr) {
 460                 return RESUME_GUEST;
 461         }
 462
 463 #ifdef DEBUG_EXT
 464         printk(KERN_INFO "Loading up ext 0x%lx\n", msr);
 465 #endif
 466
 467         current->thread.regs->msr |= msr;
 468
 469         switch (msr) {
 470         case MSR_FP:
 471                 for (i = 0; i < ARRAY_SIZE(vcpu->arch.fpr); i++)
 472                         thread_fpr[get_fpr_index(i)] = vcpu_fpr[i];
 473
 474                 t->fpscr.val = vcpu->arch.fpscr;
 475                 t->fpexc_mode = 0;
 476                 kvmppc_load_up_fpu();
 477                 break;
 478         case MSR_VEC:
 479 #ifdef CONFIG_ALTIVEC
 480                 memcpy(t->vr, vcpu->arch.vr, sizeof(vcpu->arch.vr));
 481                 t->vscr = vcpu->arch.vscr;
 482                 t->vrsave = -1;
 483                 kvmppc_load_up_altivec();
 484 #endif
 485                 break;
 486         case MSR_VSX:
 487 #ifdef CONFIG_VSX
 488                 for (i = 0; i < ARRAY_SIZE(vcpu->arch.vsr); i++)
 489                         thread_fpr[get_fpr_index(i) + 1] = vcpu_vsx[i];
 490                 kvmppc_load_up_vsx();
 491 #endif
 492                 break;
 493         default:
 494                 BUG();
 495         }
 496
 497         vcpu->arch.guest_owned_ext |= msr;
 498
 499         kvmppc_recalc_shadow_msr(vcpu);
 500
 501         return RESUME_GUEST;
 502 }
 503
 504 int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu,
 505                        unsigned int exit_nr)
 506 {
 507         int r = RESUME_HOST;
 508
 509         vcpu->stat.sum_exits++;
 510
 511         run->exit_reason = KVM_EXIT_UNKNOWN;
 512         run->ready_for_interrupt_injection = 1;
 513
 514         trace_kvm_book3s_exit(exit_nr, vcpu);
 515         kvm_resched(vcpu);
 516         switch (exit_nr) {
 517         case BOOK3S_INTERRUPT_INST_STORAGE:
 518                 vcpu->stat.pf_instruc++;
 519
 520 #ifdef CONFIG_PPC_BOOK3S_32
 521                 /* We set segments as unused segments when invalidating them. So
 522                  * treat the respective fault as segment fault. */
 523                 if (to_svcpu(vcpu)->sr[kvmppc_get_pc(vcpu) >> SID_SHIFT]
 524                     == SR_INVALID) {
 525                         kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu));
 526                         r = RESUME_GUEST;
 527                         break;
 528                 }
 529 #endif
 530
 531                 /* only care about PTEG not found errors, but leave NX alone */
 532                 if (to_svcpu(vcpu)->shadow_srr1 & 0x40000000) {
 533                         r = kvmppc_handle_pagefault(run, vcpu, kvmppc_get_pc(vcpu), exit_nr);
 534                         vcpu->stat.sp_instruc++;
 535                 } else if (vcpu->arch.mmu.is_dcbz32(vcpu) &&
 536                           (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32))) {
 537                         /*
 538                          * XXX If we do the dcbz hack we use the NX bit to flush&patch the page,
 539                          *     so we can't use the NX bit inside the guest. Let's cross our fingers,
 540                          *     that no guest that needs the dcbz hack does NX.
 541                          */
 542                         kvmppc_mmu_pte_flush(vcpu, kvmppc_get_pc(vcpu), ~0xFFFUL);
 543                         r = RESUME_GUEST;
 544                 } else {
 545                         vcpu->arch.shared->msr |=
 546                                 to_svcpu(vcpu)->shadow_srr1 & 0x58000000;
 547                         kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
 548                         r = RESUME_GUEST;
 549                 }
 550                 break;
 551         case BOOK3S_INTERRUPT_DATA_STORAGE:
 552         {
 553                 ulong dar = kvmppc_get_fault_dar(vcpu);
 554                 vcpu->stat.pf_storage++;
 555
 556 #ifdef CONFIG_PPC_BOOK3S_32
 557                 /* We set segments as unused segments when invalidating them. So
 558                  * treat the respective fault as segment fault. */
 559                 if ((to_svcpu(vcpu)->sr[dar >> SID_SHIFT]) == SR_INVALID) {
 560                         kvmppc_mmu_map_segment(vcpu, dar);
 561                         r = RESUME_GUEST;
 562                         break;
 563                 }
 564 #endif
 565
 566                 /* The only case we need to handle is missing shadow PTEs */
 567                 if (to_svcpu(vcpu)->fault_dsisr & DSISR_NOHPTE) {
 568                         r = kvmppc_handle_pagefault(run, vcpu, dar, exit_nr);
 569                 } else {
 570                         vcpu->arch.shared->dar = dar;
 571                         vcpu->arch.shared->dsisr = to_svcpu(vcpu)->fault_dsisr;
 572                         kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
 573                         r = RESUME_GUEST;
 574                 }
 575                 break;
 576         }
 577         case BOOK3S_INTERRUPT_DATA_SEGMENT:
 578                 if (kvmppc_mmu_map_segment(vcpu, kvmppc_get_fault_dar(vcpu)) < 0) {
 579                         vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu);
 580                         kvmppc_book3s_queue_irqprio(vcpu,
 581                                 BOOK3S_INTERRUPT_DATA_SEGMENT);
 582                 }
 583                 r = RESUME_GUEST;
 584                 break;
 585         case BOOK3S_INTERRUPT_INST_SEGMENT:
 586                 if (kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu)) < 0) {
 587                         kvmppc_book3s_queue_irqprio(vcpu,
 588                                 BOOK3S_INTERRUPT_INST_SEGMENT);
 589                 }
 590                 r = RESUME_GUEST;
 591                 break;
 592         /* We're good on these - the host merely wanted to get our attention */
 593         case BOOK3S_INTERRUPT_DECREMENTER:
 594                 vcpu->stat.dec_exits++;
 595                 r = RESUME_GUEST;
 596                 break;
 597         case BOOK3S_INTERRUPT_EXTERNAL:
 598                 vcpu->stat.ext_intr_exits++;
 599                 r = RESUME_GUEST;
 600                 break;
 601         case BOOK3S_INTERRUPT_PERFMON:
 602                 r = RESUME_GUEST;
 603                 break;
 604         case BOOK3S_INTERRUPT_PROGRAM:
 605         {
 606                 enum emulation_result er;
 607                 ulong flags;
 608
 609 program_interrupt:
 610                 flags = to_svcpu(vcpu)->shadow_srr1 & 0x1f0000ull;
 611
 612                 if (vcpu->arch.shared->msr & MSR_PR) {
 613 #ifdef EXIT_DEBUG
 614                         printk(KERN_INFO "Userspace triggered 0x700 exception at 0x%lx (0x%x)\n", kvmppc_get_pc(vcpu), kvmppc_get_last_inst(vcpu));
 615 #endif
 616                         if ((kvmppc_get_last_inst(vcpu) & 0xff0007ff) !=
 617                             (INS_DCBZ & 0xfffffff7)) {
 618                                 kvmppc_core_queue_program(vcpu, flags);
 619                                 r = RESUME_GUEST;
 620                                 break;
 621                         }
 622                 }
 623
 624                 vcpu->stat.emulated_inst_exits++;
 625                 er = kvmppc_emulate_instruction(run, vcpu);
 626                 switch (er) {
 627                 case EMULATE_DONE:
 628                         r = RESUME_GUEST_NV;
 629                         break;
 630                 case EMULATE_AGAIN:
 631                         r = RESUME_GUEST;
 632                         break;
 633                 case EMULATE_FAIL:
 634                         printk(KERN_CRIT "%s: emulation at %lx failed (%08x)\n",
 635                                __func__, kvmppc_get_pc(vcpu), kvmppc_get_last_inst(vcpu));
 636                         kvmppc_core_queue_program(vcpu, flags);
 637                         r = RESUME_GUEST;
 638                         break;
 639                 case EMULATE_DO_MMIO:
 640                         run->exit_reason = KVM_EXIT_MMIO;
 641                         r = RESUME_HOST_NV;
 642                         break;
 643                 default:
 644                         BUG();
 645                 }
 646                 break;
 647         }
 648         case BOOK3S_INTERRUPT_SYSCALL:
 649                 if (vcpu->arch.osi_enabled &&
 650                     (((u32)kvmppc_get_gpr(vcpu, 3)) == OSI_SC_MAGIC_R3) &&
 651                     (((u32)kvmppc_get_gpr(vcpu, 4)) == OSI_SC_MAGIC_R4)) {
 652                         /* MOL hypercalls */
 653                         u64 *gprs = run->osi.gprs;
 654                         int i;
 655
 656                         run->exit_reason = KVM_EXIT_OSI;
 657                         for (i = 0; i < 32; i++)
 658                                 gprs[i] = kvmppc_get_gpr(vcpu, i);
 659                         vcpu->arch.osi_needed = 1;
 660                         r = RESUME_HOST_NV;
 661                 } else if (!(vcpu->arch.shared->msr & MSR_PR) &&
 662                     (((u32)kvmppc_get_gpr(vcpu, 0)) == KVM_SC_MAGIC_R0)) {
 663                         /* KVM PV hypercalls */
 664                         kvmppc_set_gpr(vcpu, 3, kvmppc_kvm_pv(vcpu));
 665                         r = RESUME_GUEST;
 666                 } else {
 667                         /* Guest syscalls */
 668                         vcpu->stat.syscall_exits++;
 669                         kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
 670                         r = RESUME_GUEST;
 671                 }
 672                 break;
 673         case BOOK3S_INTERRUPT_FP_UNAVAIL:
 674         case BOOK3S_INTERRUPT_ALTIVEC:
 675         case BOOK3S_INTERRUPT_VSX:
 676         {
 677                 int ext_msr = 0;
 678
 679                 switch (exit_nr) {
 680                 case BOOK3S_INTERRUPT_FP_UNAVAIL: ext_msr = MSR_FP;  break;
 681                 case BOOK3S_INTERRUPT_ALTIVEC:    ext_msr = MSR_VEC; break;
 682                 case BOOK3S_INTERRUPT_VSX:        ext_msr = MSR_VSX; break;
 683                 }
 684
 685                 switch (kvmppc_check_ext(vcpu, exit_nr)) {
 686                 case EMULATE_DONE:
 687                         /* everything ok - let's enable the ext */
 688                         r = kvmppc_handle_ext(vcpu, exit_nr, ext_msr);
 689                         break;
 690                 case EMULATE_FAIL:
 691                         /* we need to emulate this instruction */
 692                         goto program_interrupt;
 693                         break;
 694                 default:
 695                         /* nothing to worry about - go again */
 696                         break;
 697                 }
 698                 break;
 699         }
 700         case BOOK3S_INTERRUPT_ALIGNMENT:
 701                 if (kvmppc_read_inst(vcpu) == EMULATE_DONE) {
 702                         vcpu->arch.shared->dsisr = kvmppc_alignment_dsisr(vcpu,
 703                                 kvmppc_get_last_inst(vcpu));
 704                         vcpu->arch.shared->dar = kvmppc_alignment_dar(vcpu,
 705                                 kvmppc_get_last_inst(vcpu));
 706                         kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
 707                 }
 708                 r = RESUME_GUEST;
 709                 break;
 710         case BOOK3S_INTERRUPT_MACHINE_CHECK:
 711         case BOOK3S_INTERRUPT_TRACE:
 712                 kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
 713                 r = RESUME_GUEST;
 714                 break;
 715         default:
 716                 /* Ugh - bork here! What did we get? */
 717                 printk(KERN_EMERG "exit_nr=0x%x | pc=0x%lx | msr=0x%lx\n",
 718                         exit_nr, kvmppc_get_pc(vcpu), to_svcpu(vcpu)->shadow_srr1);
 719                 r = RESUME_HOST;
 720                 BUG();
 721                 break;
 722         }
 723
 724
 725         if (!(r & RESUME_HOST)) {
 726                 /* To avoid clobbering exit_reason, only check for signals if
 727                  * we aren't already exiting to userspace for some other
 728                  * reason. */
 729                 if (signal_pending(current)) {
 730 #ifdef EXIT_DEBUG
 731                         printk(KERN_EMERG "KVM: Going back to host\n");
 732 #endif
 733                         vcpu->stat.signal_exits++;
 734                         run->exit_reason = KVM_EXIT_INTR;
 735                         r = -EINTR;
 736                 } else {
 737                         /* In case an interrupt came in that was triggered
 738                          * from userspace (like DEC), we need to check what
 739                          * to inject now! */
 740                         kvmppc_core_deliver_interrupts(vcpu);
 741                 }
 742         }
 743
 744         trace_kvm_book3s_reenter(r, vcpu);
 745
 746         return r;
 747 }
 748
 749 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
 750                                   struct kvm_sregs *sregs)
 751 {
 752         struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu);
 753         int i;
 754
 755         sregs->pvr = vcpu->arch.pvr;
 756
 757         sregs->u.s.sdr1 = to_book3s(vcpu)->sdr1;
 758         if (vcpu->arch.hflags & BOOK3S_HFLAG_SLB) {
 759                 for (i = 0; i < 64; i++) {
 760                         sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige | i;
 761                         sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv;
 762                 }
 763         } else {
 764                 for (i = 0; i < 16; i++)
 765                         sregs->u.s.ppc32.sr[i] = vcpu->arch.shared->sr[i];
 766
 767                 for (i = 0; i < 8; i++) {
 768                         sregs->u.s.ppc32.ibat[i] = vcpu3s->ibat[i].raw;
 769                         sregs->u.s.ppc32.dbat[i] = vcpu3s->dbat[i].raw;
 770                 }
 771         }
 772
 773         return 0;
 774 }
 775
 776 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
 777                                   struct kvm_sregs *sregs)
 778 {
 779         struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu);
 780         int i;
 781
 782         kvmppc_set_pvr(vcpu, sregs->pvr);
 783
 784         vcpu3s->sdr1 = sregs->u.s.sdr1;
 785         if (vcpu->arch.hflags & BOOK3S_HFLAG_SLB) {
 786                 for (i = 0; i < 64; i++) {
 787                         vcpu->arch.mmu.slbmte(vcpu, sregs->u.s.ppc64.slb[i].slbv,
 788                                                     sregs->u.s.ppc64.slb[i].slbe);
 789                 }
 790         } else {
 791                 for (i = 0; i < 16; i++) {
 792                         vcpu->arch.mmu.mtsrin(vcpu, i, sregs->u.s.ppc32.sr[i]);
 793                 }
 794                 for (i = 0; i < 8; i++) {
 795                         kvmppc_set_bat(vcpu, &(vcpu3s->ibat[i]), false,
 796                                        (u32)sregs->u.s.ppc32.ibat[i]);
 797                         kvmppc_set_bat(vcpu, &(vcpu3s->ibat[i]), true,
 798                                        (u32)(sregs->u.s.ppc32.ibat[i] >> 32));
 799                         kvmppc_set_bat(vcpu, &(vcpu3s->dbat[i]), false,
 800                                        (u32)sregs->u.s.ppc32.dbat[i]);
 801                         kvmppc_set_bat(vcpu, &(vcpu3s->dbat[i]), true,
 802                                        (u32)(sregs->u.s.ppc32.dbat[i] >> 32));
 803                 }
 804         }
 805
 806         /* Flush the MMU after messing with the segments */
 807         kvmppc_mmu_pte_flush(vcpu, 0, 0);
 808
 809         return 0;
 810 }
 811
 812 int kvmppc_core_check_processor_compat(void)
 813 {
 814         return 0;
 815 }
 816
 817 struct kvm_vcpu *kvmppc_core_vcpu_create(struct kvm *kvm, unsigned int id)
 818 {
 819         struct kvmppc_vcpu_book3s *vcpu_book3s;
 820         struct kvm_vcpu *vcpu;
 821         int err = -ENOMEM;
 822         unsigned long p;
 823
 824         vcpu_book3s = vzalloc(sizeof(struct kvmppc_vcpu_book3s));
 825         if (!vcpu_book3s)
 826                 goto out;
 827
 828         vcpu_book3s->shadow_vcpu = (struct kvmppc_book3s_shadow_vcpu *)
 829                 kzalloc(sizeof(*vcpu_book3s->shadow_vcpu), GFP_KERNEL);
 830         if (!vcpu_book3s->shadow_vcpu)
 831                 goto free_vcpu;
 832
 833         vcpu = &vcpu_book3s->vcpu;
 834         err = kvm_vcpu_init(vcpu, kvm, id);
 835         if (err)
 836                 goto free_shadow_vcpu;
 837
 838         p = __get_free_page(GFP_KERNEL|__GFP_ZERO);
 839         /* the real shared page fills the last 4k of our page */
 840         vcpu->arch.shared = (void*)(p + PAGE_SIZE - 4096);
 841         if (!p)
 842                 goto uninit_vcpu;
 843
 844         vcpu->arch.host_retip = kvm_return_point;
 845         vcpu->arch.host_msr = mfmsr();
 846 #ifdef CONFIG_PPC_BOOK3S_64
 847         /* default to book3s_64 (970fx) */
 848         vcpu->arch.pvr = 0x3C0301;
 849 #else
 850         /* default to book3s_32 (750) */
 851         vcpu->arch.pvr = 0x84202;
 852 #endif
 853         kvmppc_set_pvr(vcpu, vcpu->arch.pvr);
 854         vcpu->arch.slb_nr = 64;
 855
 856         /* remember where some real-mode handlers are */
 857         vcpu->arch.trampoline_lowmem = __pa(kvmppc_handler_lowmem_trampoline);
 858         vcpu->arch.trampoline_enter = __pa(kvmppc_handler_trampoline_enter);
 859         vcpu->arch.highmem_handler = (ulong)kvmppc_handler_highmem;
 860 #ifdef CONFIG_PPC_BOOK3S_64
 861         vcpu->arch.rmcall = *(ulong*)kvmppc_rmcall;
 862 #else
 863         vcpu->arch.rmcall = (ulong)kvmppc_rmcall;
 864 #endif
 865
 866         vcpu->arch.shadow_msr = MSR_USER64;
 867
 868         err = kvmppc_mmu_init(vcpu);
 869         if (err < 0)
 870                 goto uninit_vcpu;
 871
 872         return vcpu;
 873
 874 uninit_vcpu:
 875         kvm_vcpu_uninit(vcpu);
 876 free_shadow_vcpu:
 877         kfree(vcpu_book3s->shadow_vcpu);
 878 free_vcpu:
 879         vfree(vcpu_book3s);
 880 out:
 881         return ERR_PTR(err);
 882 }
 883
 884 void kvmppc_core_vcpu_free(struct kvm_vcpu *vcpu)
 885 {
 886         struct kvmppc_vcpu_book3s *vcpu_book3s = to_book3s(vcpu);
 887
 888         free_page((unsigned long)vcpu->arch.shared & PAGE_MASK);
 889         kvm_vcpu_uninit(vcpu);
 890         kfree(vcpu_book3s->shadow_vcpu);
 891         vfree(vcpu_book3s);
 892 }
 893
 894 int kvmppc_vcpu_run(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
 895 {
 896         int ret;
 897         double fpr[32][TS_FPRWIDTH];
 898         unsigned int fpscr;
 899         int fpexc_mode;
 900 #ifdef CONFIG_ALTIVEC
 901         vector128 vr[32];
 902         vector128 vscr;
 903         unsigned long uninitialized_var(vrsave);
 904         int used_vr;
 905 #endif
 906 #ifdef CONFIG_VSX
 907         int used_vsr;
 908 #endif
 909         ulong ext_msr;
 910
 911         /* No need to go into the guest when all we do is going out */
 912         if (signal_pending(current)) {
 913                 kvm_run->exit_reason = KVM_EXIT_INTR;
 914                 return -EINTR;
 915         }
 916
 917         /* Save FPU state in stack */
 918         if (current->thread.regs->msr & MSR_FP)
 919                 giveup_fpu(current);
 920         memcpy(fpr, current->thread.fpr, sizeof(current->thread.fpr));
 921         fpscr = current->thread.fpscr.val;
 922         fpexc_mode = current->thread.fpexc_mode;
 923
 924 #ifdef CONFIG_ALTIVEC
 925         /* Save Altivec state in stack */
 926         used_vr = current->thread.used_vr;
 927         if (used_vr) {
 928                 if (current->thread.regs->msr & MSR_VEC)
 929                         giveup_altivec(current);
 930                 memcpy(vr, current->thread.vr, sizeof(current->thread.vr));
 931                 vscr = current->thread.vscr;
 932                 vrsave = current->thread.vrsave;
 933         }
 934 #endif
 935
 936 #ifdef CONFIG_VSX
 937         /* Save VSX state in stack */
 938         used_vsr = current->thread.used_vsr;
 939         if (used_vsr && (current->thread.regs->msr & MSR_VSX))
 940                         __giveup_vsx(current);
 941 #endif
 942
 943         /* Remember the MSR with disabled extensions */
 944         ext_msr = current->thread.regs->msr;
 945
 946         /* Preload FPU if it's enabled */
 947         if (vcpu->arch.shared->msr & MSR_FP)
 948                 kvmppc_handle_ext(vcpu, BOOK3S_INTERRUPT_FP_UNAVAIL, MSR_FP);
 949
 950         kvm_guest_enter();
 951
 952         ret = __kvmppc_vcpu_run(kvm_run, vcpu);
 953
 954         kvm_guest_exit();
 955
 956         local_irq_disable();
 957
 958         current->thread.regs->msr = ext_msr;
 959
 960         /* Make sure we save the guest FPU/Altivec/VSX state */
 961         kvmppc_giveup_ext(vcpu, MSR_FP);
 962         kvmppc_giveup_ext(vcpu, MSR_VEC);
 963         kvmppc_giveup_ext(vcpu, MSR_VSX);
 964
 965         /* Restore FPU state from stack */
 966         memcpy(current->thread.fpr, fpr, sizeof(current->thread.fpr));
 967         current->thread.fpscr.val = fpscr;
 968         current->thread.fpexc_mode = fpexc_mode;
 969
 970 #ifdef CONFIG_ALTIVEC
 971         /* Restore Altivec state from stack */
 972         if (used_vr && current->thread.used_vr) {
 973                 memcpy(current->thread.vr, vr, sizeof(current->thread.vr));
 974                 current->thread.vscr = vscr;
 975                 current->thread.vrsave = vrsave;
 976         }
 977         current->thread.used_vr = used_vr;
 978 #endif
 979
 980 #ifdef CONFIG_VSX
 981         current->thread.used_vsr = used_vsr;
 982 #endif
 983
 984         return ret;
 985 }
 986
 987 int kvmppc_core_prepare_memory_region(struct kvm *kvm,
 988                                       struct kvm_userspace_memory_region *mem)
 989 {
 990         return 0;
 991 }
 992
 993 void kvmppc_core_commit_memory_region(struct kvm *kvm,
 994                                 struct kvm_userspace_memory_region *mem)
 995 {
 996 }
 997
 998 int kvmppc_core_init_vm(struct kvm *kvm)
 999 {
1000         return 0;
1001 }
1002
1003 void kvmppc_core_destroy_vm(struct kvm *kvm)
1004 {
1005 }
1006
1007 static int kvmppc_book3s_init(void)
1008 {
1009         int r;
1010
1011         r = kvm_init(NULL, sizeof(struct kvmppc_vcpu_book3s), 0,
1012                      THIS_MODULE);
1013
1014         if (r)
1015                 return r;
1016
1017         r = kvmppc_mmu_hpte_sysinit();
1018
1019         return r;
1020 }
1021
1022 static void kvmppc_book3s_exit(void)
1023 {
1024         kvmppc_mmu_hpte_sysexit();
1025         kvm_exit();
1026 }
1027
1028 module_init(kvmppc_book3s_init);
1029 module_exit(kvmppc_book3s_exit);