drivers/kvm/kvm_main.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  *
   9  * Authors:
  10  *   Avi Kivity   <avi@qumranet.com>
  11  *   Yaniv Kamay  <yaniv@qumranet.com>
  12  *
  13  * This work is licensed under the terms of the GNU GPL, version 2.  See
  14  * the COPYING file in the top-level directory.
  15  *
  16  */
  17
  18 #include "kvm.h"
  19
  20 #include <linux/kvm.h>
  21 #include <linux/module.h>
  22 #include <linux/errno.h>
  23 #include <linux/magic.h>
  24 #include <asm/processor.h>
  25 #include <linux/percpu.h>
  26 #include <linux/gfp.h>
  27 #include <asm/msr.h>
  28 #include <linux/mm.h>
  29 #include <linux/miscdevice.h>
  30 #include <linux/vmalloc.h>
  31 #include <asm/uaccess.h>
  32 #include <linux/reboot.h>
  33 #include <asm/io.h>
  34 #include <linux/debugfs.h>
  35 #include <linux/highmem.h>
  36 #include <linux/file.h>
  37 #include <asm/desc.h>
  38 #include <linux/sysdev.h>
  39 #include <linux/cpu.h>
  40 #include <linux/file.h>
  41 #include <linux/fs.h>
  42 #include <linux/mount.h>
  43 #include <linux/sched.h>
  44 #include <linux/cpumask.h>
  45 #include <linux/smp.h>
  46
  47 #include "x86_emulate.h"
  48 #include "segment_descriptor.h"
  49
  50 MODULE_AUTHOR("Qumranet");
  51 MODULE_LICENSE("GPL");
  52
  53 static DEFINE_SPINLOCK(kvm_lock);
  54 static LIST_HEAD(vm_list);
  55
  56 struct kvm_arch_ops *kvm_arch_ops;
  57
  58 #define STAT_OFFSET(x) offsetof(struct kvm_vcpu, stat.x)
  59
  60 static struct kvm_stats_debugfs_item {
  61         const char *name;
  62         int offset;
  63         struct dentry *dentry;
  64 } debugfs_entries[] = {
  65         { "pf_fixed", STAT_OFFSET(pf_fixed) },
  66         { "pf_guest", STAT_OFFSET(pf_guest) },
  67         { "tlb_flush", STAT_OFFSET(tlb_flush) },
  68         { "invlpg", STAT_OFFSET(invlpg) },
  69         { "exits", STAT_OFFSET(exits) },
  70         { "io_exits", STAT_OFFSET(io_exits) },
  71         { "mmio_exits", STAT_OFFSET(mmio_exits) },
  72         { "signal_exits", STAT_OFFSET(signal_exits) },
  73         { "irq_window", STAT_OFFSET(irq_window_exits) },
  74         { "halt_exits", STAT_OFFSET(halt_exits) },
  75         { "request_irq", STAT_OFFSET(request_irq_exits) },
  76         { "irq_exits", STAT_OFFSET(irq_exits) },
  77         { "light_exits", STAT_OFFSET(light_exits) },
  78         { "efer_reload", STAT_OFFSET(efer_reload) },
  79         { NULL }
  80 };
  81
  82 static struct dentry *debugfs_dir;
  83
  84 struct vfsmount *kvmfs_mnt;
  85
  86 #define MAX_IO_MSRS 256
  87
  88 #define CR0_RESEVED_BITS 0xffffffff1ffaffc0ULL
  89 #define LMSW_GUEST_MASK 0x0eULL
  90 #define CR4_RESEVED_BITS (~((1ULL << 11) - 1))
  91 #define CR8_RESEVED_BITS (~0x0fULL)
  92 #define EFER_RESERVED_BITS 0xfffffffffffff2fe
  93
  94 #ifdef CONFIG_X86_64
  95 // LDT or TSS descriptor in the GDT. 16 bytes.
  96 struct segment_descriptor_64 {
  97         struct segment_descriptor s;
  98         u32 base_higher;
  99         u32 pad_zero;
 100 };
 101
 102 #endif
 103
 104 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
 105                            unsigned long arg);
 106
 107 static struct inode *kvmfs_inode(struct file_operations *fops)
 108 {
 109         int error = -ENOMEM;
 110         struct inode *inode = new_inode(kvmfs_mnt->mnt_sb);
 111
 112         if (!inode)
 113                 goto eexit_1;
 114
 115         inode->i_fop = fops;
 116
 117         /*
 118          * Mark the inode dirty from the very beginning,
 119          * that way it will never be moved to the dirty
 120          * list because mark_inode_dirty() will think
 121          * that it already _is_ on the dirty list.
 122          */
 123         inode->i_state = I_DIRTY;
 124         inode->i_mode = S_IRUSR | S_IWUSR;
 125         inode->i_uid = current->fsuid;
 126         inode->i_gid = current->fsgid;
 127         inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
 128         return inode;
 129
 130 eexit_1:
 131         return ERR_PTR(error);
 132 }
 133
 134 static struct file *kvmfs_file(struct inode *inode, void *private_data)
 135 {
 136         struct file *file = get_empty_filp();
 137
 138         if (!file)
 139                 return ERR_PTR(-ENFILE);
 140
 141         file->f_path.mnt = mntget(kvmfs_mnt);
 142         file->f_path.dentry = d_alloc_anon(inode);
 143         if (!file->f_path.dentry)
 144                 return ERR_PTR(-ENOMEM);
 145         file->f_mapping = inode->i_mapping;
 146
 147         file->f_pos = 0;
 148         file->f_flags = O_RDWR;
 149         file->f_op = inode->i_fop;
 150         file->f_mode = FMODE_READ | FMODE_WRITE;
 151         file->f_version = 0;
 152         file->private_data = private_data;
 153         return file;
 154 }
 155
 156 unsigned long segment_base(u16 selector)
 157 {
 158         struct descriptor_table gdt;
 159         struct segment_descriptor *d;
 160         unsigned long table_base;
 161         typedef unsigned long ul;
 162         unsigned long v;
 163
 164         if (selector == 0)
 165                 return 0;
 166
 167         asm ("sgdt %0" : "=m"(gdt));
 168         table_base = gdt.base;
 169
 170         if (selector & 4) {           /* from ldt */
 171                 u16 ldt_selector;
 172
 173                 asm ("sldt %0" : "=g"(ldt_selector));
 174                 table_base = segment_base(ldt_selector);
 175         }
 176         d = (struct segment_descriptor *)(table_base + (selector & ~7));
 177         v = d->base_low | ((ul)d->base_mid << 16) | ((ul)d->base_high << 24);
 178 #ifdef CONFIG_X86_64
 179         if (d->system == 0
 180             && (d->type == 2 || d->type == 9 || d->type == 11))
 181                 v |= ((ul)((struct segment_descriptor_64 *)d)->base_higher) << 32;
 182 #endif
 183         return v;
 184 }
 185 EXPORT_SYMBOL_GPL(segment_base);
 186
 187 static inline int valid_vcpu(int n)
 188 {
 189         return likely(n >= 0 && n < KVM_MAX_VCPUS);
 190 }
 191
 192 int kvm_read_guest(struct kvm_vcpu *vcpu, gva_t addr, unsigned long size,
 193                    void *dest)
 194 {
 195         unsigned char *host_buf = dest;
 196         unsigned long req_size = size;
 197
 198         while (size) {
 199                 hpa_t paddr;
 200                 unsigned now;
 201                 unsigned offset;
 202                 hva_t guest_buf;
 203
 204                 paddr = gva_to_hpa(vcpu, addr);
 205
 206                 if (is_error_hpa(paddr))
 207                         break;
 208
 209                 guest_buf = (hva_t)kmap_atomic(
 210                                         pfn_to_page(paddr >> PAGE_SHIFT),
 211                                         KM_USER0);
 212                 offset = addr & ~PAGE_MASK;
 213                 guest_buf |= offset;
 214                 now = min(size, PAGE_SIZE - offset);
 215                 memcpy(host_buf, (void*)guest_buf, now);
 216                 host_buf += now;
 217                 addr += now;
 218                 size -= now;
 219                 kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
 220         }
 221         return req_size - size;
 222 }
 223 EXPORT_SYMBOL_GPL(kvm_read_guest);
 224
 225 int kvm_write_guest(struct kvm_vcpu *vcpu, gva_t addr, unsigned long size,
 226                     void *data)
 227 {
 228         unsigned char *host_buf = data;
 229         unsigned long req_size = size;
 230
 231         while (size) {
 232                 hpa_t paddr;
 233                 unsigned now;
 234                 unsigned offset;
 235                 hva_t guest_buf;
 236                 gfn_t gfn;
 237
 238                 paddr = gva_to_hpa(vcpu, addr);
 239
 240                 if (is_error_hpa(paddr))
 241                         break;
 242
 243                 gfn = vcpu->mmu.gva_to_gpa(vcpu, addr) >> PAGE_SHIFT;
 244                 mark_page_dirty(vcpu->kvm, gfn);
 245                 guest_buf = (hva_t)kmap_atomic(
 246                                 pfn_to_page(paddr >> PAGE_SHIFT), KM_USER0);
 247                 offset = addr & ~PAGE_MASK;
 248                 guest_buf |= offset;
 249                 now = min(size, PAGE_SIZE - offset);
 250                 memcpy((void*)guest_buf, host_buf, now);
 251                 host_buf += now;
 252                 addr += now;
 253                 size -= now;
 254                 kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
 255         }
 256         return req_size - size;
 257 }
 258 EXPORT_SYMBOL_GPL(kvm_write_guest);
 259
 260 void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
 261 {
 262         if (!vcpu->fpu_active || vcpu->guest_fpu_loaded)
 263                 return;
 264
 265         vcpu->guest_fpu_loaded = 1;
 266         fx_save(vcpu->host_fx_image);
 267         fx_restore(vcpu->guest_fx_image);
 268 }
 269 EXPORT_SYMBOL_GPL(kvm_load_guest_fpu);
 270
 271 void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
 272 {
 273         if (!vcpu->guest_fpu_loaded)
 274                 return;
 275
 276         vcpu->guest_fpu_loaded = 0;
 277         fx_save(vcpu->guest_fx_image);
 278         fx_restore(vcpu->host_fx_image);
 279 }
 280 EXPORT_SYMBOL_GPL(kvm_put_guest_fpu);
 281
 282 /*
 283  * Switches to specified vcpu, until a matching vcpu_put()
 284  */
 285 static void vcpu_load(struct kvm_vcpu *vcpu)
 286 {
 287         mutex_lock(&vcpu->mutex);
 288         kvm_arch_ops->vcpu_load(vcpu);
 289 }
 290
 291 /*
 292  * Switches to specified vcpu, until a matching vcpu_put(). Will return NULL
 293  * if the slot is not populated.
 294  */
 295 static struct kvm_vcpu *vcpu_load_slot(struct kvm *kvm, int slot)
 296 {
 297         struct kvm_vcpu *vcpu = &kvm->vcpus[slot];
 298
 299         mutex_lock(&vcpu->mutex);
 300         if (!vcpu->vmcs) {
 301                 mutex_unlock(&vcpu->mutex);
 302                 return NULL;
 303         }
 304         kvm_arch_ops->vcpu_load(vcpu);
 305         return vcpu;
 306 }
 307
 308 static void vcpu_put(struct kvm_vcpu *vcpu)
 309 {
 310         kvm_arch_ops->vcpu_put(vcpu);
 311         mutex_unlock(&vcpu->mutex);
 312 }
 313
 314 static void ack_flush(void *_completed)
 315 {
 316         atomic_t *completed = _completed;
 317
 318         atomic_inc(completed);
 319 }
 320
 321 void kvm_flush_remote_tlbs(struct kvm *kvm)
 322 {
 323         int i, cpu, needed;
 324         cpumask_t cpus;
 325         struct kvm_vcpu *vcpu;
 326         atomic_t completed;
 327
 328         atomic_set(&completed, 0);
 329         cpus_clear(cpus);
 330         needed = 0;
 331         for (i = 0; i < kvm->nvcpus; ++i) {
 332                 vcpu = &kvm->vcpus[i];
 333                 if (test_and_set_bit(KVM_TLB_FLUSH, &vcpu->requests))
 334                         continue;
 335                 cpu = vcpu->cpu;
 336                 if (cpu != -1 && cpu != raw_smp_processor_id())
 337                         if (!cpu_isset(cpu, cpus)) {
 338                                 cpu_set(cpu, cpus);
 339                                 ++needed;
 340                         }
 341         }
 342
 343         /*
 344          * We really want smp_call_function_mask() here.  But that's not
 345          * available, so ipi all cpus in parallel and wait for them
 346          * to complete.
 347          */
 348         for (cpu = first_cpu(cpus); cpu != NR_CPUS; cpu = next_cpu(cpu, cpus))
 349                 smp_call_function_single(cpu, ack_flush, &completed, 1, 0);
 350         while (atomic_read(&completed) != needed) {
 351                 cpu_relax();
 352                 barrier();
 353         }
 354 }
 355
 356 static struct kvm *kvm_create_vm(void)
 357 {
 358         struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
 359         int i;
 360
 361         if (!kvm)
 362                 return ERR_PTR(-ENOMEM);
 363
 364         spin_lock_init(&kvm->lock);
 365         INIT_LIST_HEAD(&kvm->active_mmu_pages);
 366         spin_lock(&kvm_lock);
 367         list_add(&kvm->vm_list, &vm_list);
 368         spin_unlock(&kvm_lock);
 369         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 370                 struct kvm_vcpu *vcpu = &kvm->vcpus[i];
 371
 372                 mutex_init(&vcpu->mutex);
 373                 vcpu->cpu = -1;
 374                 vcpu->kvm = kvm;
 375                 vcpu->mmu.root_hpa = INVALID_PAGE;
 376         }
 377         return kvm;
 378 }
 379
 380 static int kvm_dev_open(struct inode *inode, struct file *filp)
 381 {
 382         return 0;
 383 }
 384
 385 /*
 386  * Free any memory in @free but not in @dont.
 387  */
 388 static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
 389                                   struct kvm_memory_slot *dont)
 390 {
 391         int i;
 392
 393         if (!dont || free->phys_mem != dont->phys_mem)
 394                 if (free->phys_mem) {
 395                         for (i = 0; i < free->npages; ++i)
 396                                 if (free->phys_mem[i])
 397                                         __free_page(free->phys_mem[i]);
 398                         vfree(free->phys_mem);
 399                 }
 400
 401         if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
 402                 vfree(free->dirty_bitmap);
 403
 404         free->phys_mem = NULL;
 405         free->npages = 0;
 406         free->dirty_bitmap = NULL;
 407 }
 408
 409 static void kvm_free_physmem(struct kvm *kvm)
 410 {
 411         int i;
 412
 413         for (i = 0; i < kvm->nmemslots; ++i)
 414                 kvm_free_physmem_slot(&kvm->memslots[i], NULL);
 415 }
 416
 417 static void free_pio_guest_pages(struct kvm_vcpu *vcpu)
 418 {
 419         int i;
 420
 421         for (i = 0; i < 2; ++i)
 422                 if (vcpu->pio.guest_pages[i]) {
 423                         __free_page(vcpu->pio.guest_pages[i]);
 424                         vcpu->pio.guest_pages[i] = NULL;
 425                 }
 426 }
 427
 428 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
 429 {
 430         if (!vcpu->vmcs)
 431                 return;
 432
 433         vcpu_load(vcpu);
 434         kvm_mmu_unload(vcpu);
 435         vcpu_put(vcpu);
 436 }
 437
 438 static void kvm_free_vcpu(struct kvm_vcpu *vcpu)
 439 {
 440         if (!vcpu->vmcs)
 441                 return;
 442
 443         vcpu_load(vcpu);
 444         kvm_mmu_destroy(vcpu);
 445         vcpu_put(vcpu);
 446         kvm_arch_ops->vcpu_free(vcpu);
 447         free_page((unsigned long)vcpu->run);
 448         vcpu->run = NULL;
 449         free_page((unsigned long)vcpu->pio_data);
 450         vcpu->pio_data = NULL;
 451         free_pio_guest_pages(vcpu);
 452 }
 453
 454 static void kvm_free_vcpus(struct kvm *kvm)
 455 {
 456         unsigned int i;
 457
 458         /*
 459          * Unpin any mmu pages first.
 460          */
 461         for (i = 0; i < KVM_MAX_VCPUS; ++i)
 462                 kvm_unload_vcpu_mmu(&kvm->vcpus[i]);
 463         for (i = 0; i < KVM_MAX_VCPUS; ++i)
 464                 kvm_free_vcpu(&kvm->vcpus[i]);
 465 }
 466
 467 static int kvm_dev_release(struct inode *inode, struct file *filp)
 468 {
 469         return 0;
 470 }
 471
 472 static void kvm_destroy_vm(struct kvm *kvm)
 473 {
 474         spin_lock(&kvm_lock);
 475         list_del(&kvm->vm_list);
 476         spin_unlock(&kvm_lock);
 477         kvm_free_vcpus(kvm);
 478         kvm_free_physmem(kvm);
 479         kfree(kvm);
 480 }
 481
 482 static int kvm_vm_release(struct inode *inode, struct file *filp)
 483 {
 484         struct kvm *kvm = filp->private_data;
 485
 486         kvm_destroy_vm(kvm);
 487         return 0;
 488 }
 489
 490 static void inject_gp(struct kvm_vcpu *vcpu)
 491 {
 492         kvm_arch_ops->inject_gp(vcpu, 0);
 493 }
 494
 495 /*
 496  * Load the pae pdptrs.  Return true is they are all valid.
 497  */
 498 static int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
 499 {
 500         gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
 501         unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
 502         int i;
 503         u64 pdpte;
 504         u64 *pdpt;
 505         int ret;
 506         struct page *page;
 507
 508         spin_lock(&vcpu->kvm->lock);
 509         page = gfn_to_page(vcpu->kvm, pdpt_gfn);
 510         /* FIXME: !page - emulate? 0xff? */
 511         pdpt = kmap_atomic(page, KM_USER0);
 512
 513         ret = 1;
 514         for (i = 0; i < 4; ++i) {
 515                 pdpte = pdpt[offset + i];
 516                 if ((pdpte & 1) && (pdpte & 0xfffffff0000001e6ull)) {
 517                         ret = 0;
 518                         goto out;
 519                 }
 520         }
 521
 522         for (i = 0; i < 4; ++i)
 523                 vcpu->pdptrs[i] = pdpt[offset + i];
 524
 525 out:
 526         kunmap_atomic(pdpt, KM_USER0);
 527         spin_unlock(&vcpu->kvm->lock);
 528
 529         return ret;
 530 }
 531
 532 void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 533 {
 534         if (cr0 & CR0_RESEVED_BITS) {
 535                 printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
 536                        cr0, vcpu->cr0);
 537                 inject_gp(vcpu);
 538                 return;
 539         }
 540
 541         if ((cr0 & CR0_NW_MASK) && !(cr0 & CR0_CD_MASK)) {
 542                 printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n");
 543                 inject_gp(vcpu);
 544                 return;
 545         }
 546
 547         if ((cr0 & CR0_PG_MASK) && !(cr0 & CR0_PE_MASK)) {
 548                 printk(KERN_DEBUG "set_cr0: #GP, set PG flag "
 549                        "and a clear PE flag\n");
 550                 inject_gp(vcpu);
 551                 return;
 552         }
 553
 554         if (!is_paging(vcpu) && (cr0 & CR0_PG_MASK)) {
 555 #ifdef CONFIG_X86_64
 556                 if ((vcpu->shadow_efer & EFER_LME)) {
 557                         int cs_db, cs_l;
 558
 559                         if (!is_pae(vcpu)) {
 560                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
 561                                        "in long mode while PAE is disabled\n");
 562                                 inject_gp(vcpu);
 563                                 return;
 564                         }
 565                         kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
 566                         if (cs_l) {
 567                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
 568                                        "in long mode while CS.L == 1\n");
 569                                 inject_gp(vcpu);
 570                                 return;
 571
 572                         }
 573                 } else
 574 #endif
 575                 if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) {
 576                         printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
 577                                "reserved bits\n");
 578                         inject_gp(vcpu);
 579                         return;
 580                 }
 581
 582         }
 583
 584         kvm_arch_ops->set_cr0(vcpu, cr0);
 585         vcpu->cr0 = cr0;
 586
 587         spin_lock(&vcpu->kvm->lock);
 588         kvm_mmu_reset_context(vcpu);
 589         spin_unlock(&vcpu->kvm->lock);
 590         return;
 591 }
 592 EXPORT_SYMBOL_GPL(set_cr0);
 593
 594 void lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
 595 {
 596         set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f));
 597 }
 598 EXPORT_SYMBOL_GPL(lmsw);
 599
 600 void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 601 {
 602         if (cr4 & CR4_RESEVED_BITS) {
 603                 printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n");
 604                 inject_gp(vcpu);
 605                 return;
 606         }
 607
 608         if (is_long_mode(vcpu)) {
 609                 if (!(cr4 & CR4_PAE_MASK)) {
 610                         printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while "
 611                                "in long mode\n");
 612                         inject_gp(vcpu);
 613                         return;
 614                 }
 615         } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & CR4_PAE_MASK)
 616                    && !load_pdptrs(vcpu, vcpu->cr3)) {
 617                 printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
 618                 inject_gp(vcpu);
 619         }
 620
 621         if (cr4 & CR4_VMXE_MASK) {
 622                 printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n");
 623                 inject_gp(vcpu);
 624                 return;
 625         }
 626         kvm_arch_ops->set_cr4(vcpu, cr4);
 627         spin_lock(&vcpu->kvm->lock);
 628         kvm_mmu_reset_context(vcpu);
 629         spin_unlock(&vcpu->kvm->lock);
 630 }
 631 EXPORT_SYMBOL_GPL(set_cr4);
 632
 633 void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
 634 {
 635         if (is_long_mode(vcpu)) {
 636                 if (cr3 & CR3_L_MODE_RESEVED_BITS) {
 637                         printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
 638                         inject_gp(vcpu);
 639                         return;
 640                 }
 641         } else {
 642                 if (cr3 & CR3_RESEVED_BITS) {
 643                         printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
 644                         inject_gp(vcpu);
 645                         return;
 646                 }
 647                 if (is_paging(vcpu) && is_pae(vcpu) &&
 648                     !load_pdptrs(vcpu, cr3)) {
 649                         printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
 650                                "reserved bits\n");
 651                         inject_gp(vcpu);
 652                         return;
 653                 }
 654         }
 655
 656         vcpu->cr3 = cr3;
 657         spin_lock(&vcpu->kvm->lock);
 658         /*
 659          * Does the new cr3 value map to physical memory? (Note, we
 660          * catch an invalid cr3 even in real-mode, because it would
 661          * cause trouble later on when we turn on paging anyway.)
 662          *
 663          * A real CPU would silently accept an invalid cr3 and would
 664          * attempt to use it - with largely undefined (and often hard
 665          * to debug) behavior on the guest side.
 666          */
 667         if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
 668                 inject_gp(vcpu);
 669         else
 670                 vcpu->mmu.new_cr3(vcpu);
 671         spin_unlock(&vcpu->kvm->lock);
 672 }
 673 EXPORT_SYMBOL_GPL(set_cr3);
 674
 675 void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
 676 {
 677         if ( cr8 & CR8_RESEVED_BITS) {
 678                 printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8);
 679                 inject_gp(vcpu);
 680                 return;
 681         }
 682         vcpu->cr8 = cr8;
 683 }
 684 EXPORT_SYMBOL_GPL(set_cr8);
 685
 686 void fx_init(struct kvm_vcpu *vcpu)
 687 {
 688         struct __attribute__ ((__packed__)) fx_image_s {
 689                 u16 control; //fcw
 690                 u16 status; //fsw
 691                 u16 tag; // ftw
 692                 u16 opcode; //fop
 693                 u64 ip; // fpu ip
 694                 u64 operand;// fpu dp
 695                 u32 mxcsr;
 696                 u32 mxcsr_mask;
 697
 698         } *fx_image;
 699
 700         fx_save(vcpu->host_fx_image);
 701         fpu_init();
 702         fx_save(vcpu->guest_fx_image);
 703         fx_restore(vcpu->host_fx_image);
 704
 705         fx_image = (struct fx_image_s *)vcpu->guest_fx_image;
 706         fx_image->mxcsr = 0x1f80;
 707         memset(vcpu->guest_fx_image + sizeof(struct fx_image_s),
 708                0, FX_IMAGE_SIZE - sizeof(struct fx_image_s));
 709 }
 710 EXPORT_SYMBOL_GPL(fx_init);
 711
 712 static void do_remove_write_access(struct kvm_vcpu *vcpu, int slot)
 713 {
 714         spin_lock(&vcpu->kvm->lock);
 715         kvm_mmu_slot_remove_write_access(vcpu, slot);
 716         spin_unlock(&vcpu->kvm->lock);
 717 }
 718
 719 /*
 720  * Allocate some memory and give it an address in the guest physical address
 721  * space.
 722  *
 723  * Discontiguous memory is allowed, mostly for framebuffers.
 724  */
 725 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
 726                                           struct kvm_memory_region *mem)
 727 {
 728         int r;
 729         gfn_t base_gfn;
 730         unsigned long npages;
 731         unsigned long i;
 732         struct kvm_memory_slot *memslot;
 733         struct kvm_memory_slot old, new;
 734         int memory_config_version;
 735
 736         r = -EINVAL;
 737         /* General sanity checks */
 738         if (mem->memory_size & (PAGE_SIZE - 1))
 739                 goto out;
 740         if (mem->guest_phys_addr & (PAGE_SIZE - 1))
 741                 goto out;
 742         if (mem->slot >= KVM_MEMORY_SLOTS)
 743                 goto out;
 744         if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
 745                 goto out;
 746
 747         memslot = &kvm->memslots[mem->slot];
 748         base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
 749         npages = mem->memory_size >> PAGE_SHIFT;
 750
 751         if (!npages)
 752                 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
 753
 754 raced:
 755         spin_lock(&kvm->lock);
 756
 757         memory_config_version = kvm->memory_config_version;
 758         new = old = *memslot;
 759
 760         new.base_gfn = base_gfn;
 761         new.npages = npages;
 762         new.flags = mem->flags;
 763
 764         /* Disallow changing a memory slot's size. */
 765         r = -EINVAL;
 766         if (npages && old.npages && npages != old.npages)
 767                 goto out_unlock;
 768
 769         /* Check for overlaps */
 770         r = -EEXIST;
 771         for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
 772                 struct kvm_memory_slot *s = &kvm->memslots[i];
 773
 774                 if (s == memslot)
 775                         continue;
 776                 if (!((base_gfn + npages <= s->base_gfn) ||
 777                       (base_gfn >= s->base_gfn + s->npages)))
 778                         goto out_unlock;
 779         }
 780         /*
 781          * Do memory allocations outside lock.  memory_config_version will
 782          * detect any races.
 783          */
 784         spin_unlock(&kvm->lock);
 785
 786         /* Deallocate if slot is being removed */
 787         if (!npages)
 788                 new.phys_mem = NULL;
 789
 790         /* Free page dirty bitmap if unneeded */
 791         if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
 792                 new.dirty_bitmap = NULL;
 793
 794         r = -ENOMEM;
 795
 796         /* Allocate if a slot is being created */
 797         if (npages && !new.phys_mem) {
 798                 new.phys_mem = vmalloc(npages * sizeof(struct page *));
 799
 800                 if (!new.phys_mem)
 801                         goto out_free;
 802
 803                 memset(new.phys_mem, 0, npages * sizeof(struct page *));
 804                 for (i = 0; i < npages; ++i) {
 805                         new.phys_mem[i] = alloc_page(GFP_HIGHUSER
 806                                                      | __GFP_ZERO);
 807                         if (!new.phys_mem[i])
 808                                 goto out_free;
 809                         set_page_private(new.phys_mem[i],0);
 810                 }
 811         }
 812
 813         /* Allocate page dirty bitmap if needed */
 814         if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
 815                 unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;
 816
 817                 new.dirty_bitmap = vmalloc(dirty_bytes);
 818                 if (!new.dirty_bitmap)
 819                         goto out_free;
 820                 memset(new.dirty_bitmap, 0, dirty_bytes);
 821         }
 822
 823         spin_lock(&kvm->lock);
 824
 825         if (memory_config_version != kvm->memory_config_version) {
 826                 spin_unlock(&kvm->lock);
 827                 kvm_free_physmem_slot(&new, &old);
 828                 goto raced;
 829         }
 830
 831         r = -EAGAIN;
 832         if (kvm->busy)
 833                 goto out_unlock;
 834
 835         if (mem->slot >= kvm->nmemslots)
 836                 kvm->nmemslots = mem->slot + 1;
 837
 838         *memslot = new;
 839         ++kvm->memory_config_version;
 840
 841         spin_unlock(&kvm->lock);
 842
 843         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 844                 struct kvm_vcpu *vcpu;
 845
 846                 vcpu = vcpu_load_slot(kvm, i);
 847                 if (!vcpu)
 848                         continue;
 849                 if (new.flags & KVM_MEM_LOG_DIRTY_PAGES)
 850                         do_remove_write_access(vcpu, mem->slot);
 851                 kvm_mmu_reset_context(vcpu);
 852                 vcpu_put(vcpu);
 853         }
 854
 855         kvm_free_physmem_slot(&old, &new);
 856         return 0;
 857
 858 out_unlock:
 859         spin_unlock(&kvm->lock);
 860 out_free:
 861         kvm_free_physmem_slot(&new, &old);
 862 out:
 863         return r;
 864 }
 865
 866 /*
 867  * Get (and clear) the dirty memory log for a memory slot.
 868  */
 869 static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 870                                       struct kvm_dirty_log *log)
 871 {
 872         struct kvm_memory_slot *memslot;
 873         int r, i;
 874         int n;
 875         int cleared;
 876         unsigned long any = 0;
 877
 878         spin_lock(&kvm->lock);
 879
 880         /*
 881          * Prevent changes to guest memory configuration even while the lock
 882          * is not taken.
 883          */
 884         ++kvm->busy;
 885         spin_unlock(&kvm->lock);
 886         r = -EINVAL;
 887         if (log->slot >= KVM_MEMORY_SLOTS)
 888                 goto out;
 889
 890         memslot = &kvm->memslots[log->slot];
 891         r = -ENOENT;
 892         if (!memslot->dirty_bitmap)
 893                 goto out;
 894
 895         n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
 896
 897         for (i = 0; !any && i < n/sizeof(long); ++i)
 898                 any = memslot->dirty_bitmap[i];
 899
 900         r = -EFAULT;
 901         if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
 902                 goto out;
 903
 904         if (any) {
 905                 cleared = 0;
 906                 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 907                         struct kvm_vcpu *vcpu;
 908
 909                         vcpu = vcpu_load_slot(kvm, i);
 910                         if (!vcpu)
 911                                 continue;
 912                         if (!cleared) {
 913                                 do_remove_write_access(vcpu, log->slot);
 914                                 memset(memslot->dirty_bitmap, 0, n);
 915                                 cleared = 1;
 916                         }
 917                         kvm_arch_ops->tlb_flush(vcpu);
 918                         vcpu_put(vcpu);
 919                 }
 920         }
 921
 922         r = 0;
 923
 924 out:
 925         spin_lock(&kvm->lock);
 926         --kvm->busy;
 927         spin_unlock(&kvm->lock);
 928         return r;
 929 }
 930
 931 /*
 932  * Set a new alias region.  Aliases map a portion of physical memory into
 933  * another portion.  This is useful for memory windows, for example the PC
 934  * VGA region.
 935  */
 936 static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm,
 937                                          struct kvm_memory_alias *alias)
 938 {
 939         int r, n;
 940         struct kvm_mem_alias *p;
 941
 942         r = -EINVAL;
 943         /* General sanity checks */
 944         if (alias->memory_size & (PAGE_SIZE - 1))
 945                 goto out;
 946         if (alias->guest_phys_addr & (PAGE_SIZE - 1))
 947                 goto out;
 948         if (alias->slot >= KVM_ALIAS_SLOTS)
 949                 goto out;
 950         if (alias->guest_phys_addr + alias->memory_size
 951             < alias->guest_phys_addr)
 952                 goto out;
 953         if (alias->target_phys_addr + alias->memory_size
 954             < alias->target_phys_addr)
 955                 goto out;
 956
 957         spin_lock(&kvm->lock);
 958
 959         p = &kvm->aliases[alias->slot];
 960         p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT;
 961         p->npages = alias->memory_size >> PAGE_SHIFT;
 962         p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT;
 963
 964         for (n = KVM_ALIAS_SLOTS; n > 0; --n)
 965                 if (kvm->aliases[n - 1].npages)
 966                         break;
 967         kvm->naliases = n;
 968
 969         spin_unlock(&kvm->lock);
 970
 971         vcpu_load(&kvm->vcpus[0]);
 972         spin_lock(&kvm->lock);
 973         kvm_mmu_zap_all(&kvm->vcpus[0]);
 974         spin_unlock(&kvm->lock);
 975         vcpu_put(&kvm->vcpus[0]);
 976
 977         return 0;
 978
 979 out:
 980         return r;
 981 }
 982
 983 static gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
 984 {
 985         int i;
 986         struct kvm_mem_alias *alias;
 987
 988         for (i = 0; i < kvm->naliases; ++i) {
 989                 alias = &kvm->aliases[i];
 990                 if (gfn >= alias->base_gfn
 991                     && gfn < alias->base_gfn + alias->npages)
 992                         return alias->target_gfn + gfn - alias->base_gfn;
 993         }
 994         return gfn;
 995 }
 996
 997 static struct kvm_memory_slot *__gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
 998 {
 999         int i;
1000
1001         for (i = 0; i < kvm->nmemslots; ++i) {
1002                 struct kvm_memory_slot *memslot = &kvm->memslots[i];
1003
1004                 if (gfn >= memslot->base_gfn
1005                     && gfn < memslot->base_gfn + memslot->npages)
1006                         return memslot;
1007         }
1008         return NULL;
1009 }
1010
1011 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
1012 {
1013         gfn = unalias_gfn(kvm, gfn);
1014         return __gfn_to_memslot(kvm, gfn);
1015 }
1016
1017 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
1018 {
1019         struct kvm_memory_slot *slot;
1020
1021         gfn = unalias_gfn(kvm, gfn);
1022         slot = __gfn_to_memslot(kvm, gfn);
1023         if (!slot)
1024                 return NULL;
1025         return slot->phys_mem[gfn - slot->base_gfn];
1026 }
1027 EXPORT_SYMBOL_GPL(gfn_to_page);
1028
1029 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
1030 {
1031         int i;
1032         struct kvm_memory_slot *memslot;
1033         unsigned long rel_gfn;
1034
1035         for (i = 0; i < kvm->nmemslots; ++i) {
1036                 memslot = &kvm->memslots[i];
1037
1038                 if (gfn >= memslot->base_gfn
1039                     && gfn < memslot->base_gfn + memslot->npages) {
1040
1041                         if (!memslot->dirty_bitmap)
1042                                 return;
1043
1044                         rel_gfn = gfn - memslot->base_gfn;
1045
1046                         /* avoid RMW */
1047                         if (!test_bit(rel_gfn, memslot->dirty_bitmap))
1048                                 set_bit(rel_gfn, memslot->dirty_bitmap);
1049                         return;
1050                 }
1051         }
1052 }
1053
1054 static int emulator_read_std(unsigned long addr,
1055                              void *val,
1056                              unsigned int bytes,
1057                              struct x86_emulate_ctxt *ctxt)
1058 {
1059         struct kvm_vcpu *vcpu = ctxt->vcpu;
1060         void *data = val;
1061
1062         while (bytes) {
1063                 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
1064                 unsigned offset = addr & (PAGE_SIZE-1);
1065                 unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset);
1066                 unsigned long pfn;
1067                 struct page *page;
1068                 void *page_virt;
1069
1070                 if (gpa == UNMAPPED_GVA)
1071                         return X86EMUL_PROPAGATE_FAULT;
1072                 pfn = gpa >> PAGE_SHIFT;
1073                 page = gfn_to_page(vcpu->kvm, pfn);
1074                 if (!page)
1075                         return X86EMUL_UNHANDLEABLE;
1076                 page_virt = kmap_atomic(page, KM_USER0);
1077
1078                 memcpy(data, page_virt + offset, tocopy);
1079
1080                 kunmap_atomic(page_virt, KM_USER0);
1081
1082                 bytes -= tocopy;
1083                 data += tocopy;
1084                 addr += tocopy;
1085         }
1086
1087         return X86EMUL_CONTINUE;
1088 }
1089
1090 static int emulator_write_std(unsigned long addr,
1091                               const void *val,
1092                               unsigned int bytes,
1093                               struct x86_emulate_ctxt *ctxt)
1094 {
1095         printk(KERN_ERR "emulator_write_std: addr %lx n %d\n",
1096                addr, bytes);
1097         return X86EMUL_UNHANDLEABLE;
1098 }
1099
1100 static int emulator_read_emulated(unsigned long addr,
1101                                   void *val,
1102                                   unsigned int bytes,
1103                                   struct x86_emulate_ctxt *ctxt)
1104 {
1105         struct kvm_vcpu *vcpu = ctxt->vcpu;
1106
1107         if (vcpu->mmio_read_completed) {
1108                 memcpy(val, vcpu->mmio_data, bytes);
1109                 vcpu->mmio_read_completed = 0;
1110                 return X86EMUL_CONTINUE;
1111         } else if (emulator_read_std(addr, val, bytes, ctxt)
1112                    == X86EMUL_CONTINUE)
1113                 return X86EMUL_CONTINUE;
1114         else {
1115                 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
1116
1117                 if (gpa == UNMAPPED_GVA)
1118                         return X86EMUL_PROPAGATE_FAULT;
1119                 vcpu->mmio_needed = 1;
1120                 vcpu->mmio_phys_addr = gpa;
1121                 vcpu->mmio_size = bytes;
1122                 vcpu->mmio_is_write = 0;
1123
1124                 return X86EMUL_UNHANDLEABLE;
1125         }
1126 }
1127
1128 static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
1129                                const void *val, int bytes)
1130 {
1131         struct page *page;
1132         void *virt;
1133         unsigned offset = offset_in_page(gpa);
1134
1135         if (((gpa + bytes - 1) >> PAGE_SHIFT) != (gpa >> PAGE_SHIFT))
1136                 return 0;
1137         page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
1138         if (!page)
1139                 return 0;
1140         mark_page_dirty(vcpu->kvm, gpa >> PAGE_SHIFT);
1141         virt = kmap_atomic(page, KM_USER0);
1142         kvm_mmu_pte_write(vcpu, gpa, virt + offset, val, bytes);
1143         memcpy(virt + offset_in_page(gpa), val, bytes);
1144         kunmap_atomic(virt, KM_USER0);
1145         return 1;
1146 }
1147
1148 static int emulator_write_emulated(unsigned long addr,
1149                                    const void *val,
1150                                    unsigned int bytes,
1151                                    struct x86_emulate_ctxt *ctxt)
1152 {
1153         struct kvm_vcpu *vcpu = ctxt->vcpu;
1154         gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
1155
1156         if (gpa == UNMAPPED_GVA) {
1157                 kvm_arch_ops->inject_page_fault(vcpu, addr, 2);
1158                 return X86EMUL_PROPAGATE_FAULT;
1159         }
1160
1161         if (emulator_write_phys(vcpu, gpa, val, bytes))
1162                 return X86EMUL_CONTINUE;
1163
1164         vcpu->mmio_needed = 1;
1165         vcpu->mmio_phys_addr = gpa;
1166         vcpu->mmio_size = bytes;
1167         vcpu->mmio_is_write = 1;
1168         memcpy(vcpu->mmio_data, val, bytes);
1169
1170         return X86EMUL_CONTINUE;
1171 }
1172
1173 static int emulator_cmpxchg_emulated(unsigned long addr,
1174                                      const void *old,
1175                                      const void *new,
1176                                      unsigned int bytes,
1177                                      struct x86_emulate_ctxt *ctxt)
1178 {
1179         static int reported;
1180
1181         if (!reported) {
1182                 reported = 1;
1183                 printk(KERN_WARNING "kvm: emulating exchange as write\n");
1184         }
1185         return emulator_write_emulated(addr, new, bytes, ctxt);
1186 }
1187
1188 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
1189 {
1190         return kvm_arch_ops->get_segment_base(vcpu, seg);
1191 }
1192
1193 int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
1194 {
1195         return X86EMUL_CONTINUE;
1196 }
1197
1198 int emulate_clts(struct kvm_vcpu *vcpu)
1199 {
1200         unsigned long cr0;
1201
1202         cr0 = vcpu->cr0 & ~CR0_TS_MASK;
1203         kvm_arch_ops->set_cr0(vcpu, cr0);
1204         return X86EMUL_CONTINUE;
1205 }
1206
1207 int emulator_get_dr(struct x86_emulate_ctxt* ctxt, int dr, unsigned long *dest)
1208 {
1209         struct kvm_vcpu *vcpu = ctxt->vcpu;
1210
1211         switch (dr) {
1212         case 0 ... 3:
1213                 *dest = kvm_arch_ops->get_dr(vcpu, dr);
1214                 return X86EMUL_CONTINUE;
1215         default:
1216                 printk(KERN_DEBUG "%s: unexpected dr %u\n",
1217                        __FUNCTION__, dr);
1218                 return X86EMUL_UNHANDLEABLE;
1219         }
1220 }
1221
1222 int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
1223 {
1224         unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
1225         int exception;
1226
1227         kvm_arch_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception);
1228         if (exception) {
1229                 /* FIXME: better handling */
1230                 return X86EMUL_UNHANDLEABLE;
1231         }
1232         return X86EMUL_CONTINUE;
1233 }
1234
1235 static void report_emulation_failure(struct x86_emulate_ctxt *ctxt)
1236 {
1237         static int reported;
1238         u8 opcodes[4];
1239         unsigned long rip = ctxt->vcpu->rip;
1240         unsigned long rip_linear;
1241
1242         rip_linear = rip + get_segment_base(ctxt->vcpu, VCPU_SREG_CS);
1243
1244         if (reported)
1245                 return;
1246
1247         emulator_read_std(rip_linear, (void *)opcodes, 4, ctxt);
1248
1249         printk(KERN_ERR "emulation failed but !mmio_needed?"
1250                " rip %lx %02x %02x %02x %02x\n",
1251                rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]);
1252         reported = 1;
1253 }
1254
1255 struct x86_emulate_ops emulate_ops = {
1256         .read_std            = emulator_read_std,
1257         .write_std           = emulator_write_std,
1258         .read_emulated       = emulator_read_emulated,
1259         .write_emulated      = emulator_write_emulated,
1260         .cmpxchg_emulated    = emulator_cmpxchg_emulated,
1261 };
1262
1263 int emulate_instruction(struct kvm_vcpu *vcpu,
1264                         struct kvm_run *run,
1265                         unsigned long cr2,
1266                         u16 error_code)
1267 {
1268         struct x86_emulate_ctxt emulate_ctxt;
1269         int r;
1270         int cs_db, cs_l;
1271
1272         vcpu->mmio_fault_cr2 = cr2;
1273         kvm_arch_ops->cache_regs(vcpu);
1274
1275         kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
1276
1277         emulate_ctxt.vcpu = vcpu;
1278         emulate_ctxt.eflags = kvm_arch_ops->get_rflags(vcpu);
1279         emulate_ctxt.cr2 = cr2;
1280         emulate_ctxt.mode = (emulate_ctxt.eflags & X86_EFLAGS_VM)
1281                 ? X86EMUL_MODE_REAL : cs_l
1282                 ? X86EMUL_MODE_PROT64 : cs_db
1283                 ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
1284
1285         if (emulate_ctxt.mode == X86EMUL_MODE_PROT64) {
1286                 emulate_ctxt.cs_base = 0;
1287                 emulate_ctxt.ds_base = 0;
1288                 emulate_ctxt.es_base = 0;
1289                 emulate_ctxt.ss_base = 0;
1290         } else {
1291                 emulate_ctxt.cs_base = get_segment_base(vcpu, VCPU_SREG_CS);
1292                 emulate_ctxt.ds_base = get_segment_base(vcpu, VCPU_SREG_DS);
1293                 emulate_ctxt.es_base = get_segment_base(vcpu, VCPU_SREG_ES);
1294                 emulate_ctxt.ss_base = get_segment_base(vcpu, VCPU_SREG_SS);
1295         }
1296
1297         emulate_ctxt.gs_base = get_segment_base(vcpu, VCPU_SREG_GS);
1298         emulate_ctxt.fs_base = get_segment_base(vcpu, VCPU_SREG_FS);
1299
1300         vcpu->mmio_is_write = 0;
1301         r = x86_emulate_memop(&emulate_ctxt, &emulate_ops);
1302
1303         if ((r || vcpu->mmio_is_write) && run) {
1304                 run->mmio.phys_addr = vcpu->mmio_phys_addr;
1305                 memcpy(run->mmio.data, vcpu->mmio_data, 8);
1306                 run->mmio.len = vcpu->mmio_size;
1307                 run->mmio.is_write = vcpu->mmio_is_write;
1308         }
1309
1310         if (r) {
1311                 if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
1312                         return EMULATE_DONE;
1313                 if (!vcpu->mmio_needed) {
1314                         report_emulation_failure(&emulate_ctxt);
1315                         return EMULATE_FAIL;
1316                 }
1317                 return EMULATE_DO_MMIO;
1318         }
1319
1320         kvm_arch_ops->decache_regs(vcpu);
1321         kvm_arch_ops->set_rflags(vcpu, emulate_ctxt.eflags);
1322
1323         if (vcpu->mmio_is_write) {
1324                 vcpu->mmio_needed = 0;
1325                 return EMULATE_DO_MMIO;
1326         }
1327
1328         return EMULATE_DONE;
1329 }
1330 EXPORT_SYMBOL_GPL(emulate_instruction);
1331
1332 int kvm_emulate_halt(struct kvm_vcpu *vcpu)
1333 {
1334         if (vcpu->irq_summary)
1335                 return 1;
1336
1337         vcpu->run->exit_reason = KVM_EXIT_HLT;
1338         ++vcpu->stat.halt_exits;
1339         return 0;
1340 }
1341 EXPORT_SYMBOL_GPL(kvm_emulate_halt);
1342
1343 int kvm_hypercall(struct kvm_vcpu *vcpu, struct kvm_run *run)
1344 {
1345         unsigned long nr, a0, a1, a2, a3, a4, a5, ret;
1346
1347         kvm_arch_ops->cache_regs(vcpu);
1348         ret = -KVM_EINVAL;
1349 #ifdef CONFIG_X86_64
1350         if (is_long_mode(vcpu)) {
1351                 nr = vcpu->regs[VCPU_REGS_RAX];
1352                 a0 = vcpu->regs[VCPU_REGS_RDI];
1353                 a1 = vcpu->regs[VCPU_REGS_RSI];
1354                 a2 = vcpu->regs[VCPU_REGS_RDX];
1355                 a3 = vcpu->regs[VCPU_REGS_RCX];
1356                 a4 = vcpu->regs[VCPU_REGS_R8];
1357                 a5 = vcpu->regs[VCPU_REGS_R9];
1358         } else
1359 #endif
1360         {
1361                 nr = vcpu->regs[VCPU_REGS_RBX] & -1u;
1362                 a0 = vcpu->regs[VCPU_REGS_RAX] & -1u;
1363                 a1 = vcpu->regs[VCPU_REGS_RCX] & -1u;
1364                 a2 = vcpu->regs[VCPU_REGS_RDX] & -1u;
1365                 a3 = vcpu->regs[VCPU_REGS_RSI] & -1u;
1366                 a4 = vcpu->regs[VCPU_REGS_RDI] & -1u;
1367                 a5 = vcpu->regs[VCPU_REGS_RBP] & -1u;
1368         }
1369         switch (nr) {
1370         default:
1371                 run->hypercall.args[0] = a0;
1372                 run->hypercall.args[1] = a1;
1373                 run->hypercall.args[2] = a2;
1374                 run->hypercall.args[3] = a3;
1375                 run->hypercall.args[4] = a4;
1376                 run->hypercall.args[5] = a5;
1377                 run->hypercall.ret = ret;
1378                 run->hypercall.longmode = is_long_mode(vcpu);
1379                 kvm_arch_ops->decache_regs(vcpu);
1380                 return 0;
1381         }
1382         vcpu->regs[VCPU_REGS_RAX] = ret;
1383         kvm_arch_ops->decache_regs(vcpu);
1384         return 1;
1385 }
1386 EXPORT_SYMBOL_GPL(kvm_hypercall);
1387
1388 static u64 mk_cr_64(u64 curr_cr, u32 new_val)
1389 {
1390         return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
1391 }
1392
1393 void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1394 {
1395         struct descriptor_table dt = { limit, base };
1396
1397         kvm_arch_ops->set_gdt(vcpu, &dt);
1398 }
1399
1400 void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1401 {
1402         struct descriptor_table dt = { limit, base };
1403
1404         kvm_arch_ops->set_idt(vcpu, &dt);
1405 }
1406
1407 void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
1408                    unsigned long *rflags)
1409 {
1410         lmsw(vcpu, msw);
1411         *rflags = kvm_arch_ops->get_rflags(vcpu);
1412 }
1413
1414 unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
1415 {
1416         kvm_arch_ops->decache_cr4_guest_bits(vcpu);
1417         switch (cr) {
1418         case 0:
1419                 return vcpu->cr0;
1420         case 2:
1421                 return vcpu->cr2;
1422         case 3:
1423                 return vcpu->cr3;
1424         case 4:
1425                 return vcpu->cr4;
1426         default:
1427                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1428                 return 0;
1429         }
1430 }
1431
1432 void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
1433                      unsigned long *rflags)
1434 {
1435         switch (cr) {
1436         case 0:
1437                 set_cr0(vcpu, mk_cr_64(vcpu->cr0, val));
1438                 *rflags = kvm_arch_ops->get_rflags(vcpu);
1439                 break;
1440         case 2:
1441                 vcpu->cr2 = val;
1442                 break;
1443         case 3:
1444                 set_cr3(vcpu, val);
1445                 break;
1446         case 4:
1447                 set_cr4(vcpu, mk_cr_64(vcpu->cr4, val));
1448                 break;
1449         default:
1450                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1451         }
1452 }
1453
1454 /*
1455  * Register the para guest with the host:
1456  */
1457 static int vcpu_register_para(struct kvm_vcpu *vcpu, gpa_t para_state_gpa)
1458 {
1459         struct kvm_vcpu_para_state *para_state;
1460         hpa_t para_state_hpa, hypercall_hpa;
1461         struct page *para_state_page;
1462         unsigned char *hypercall;
1463         gpa_t hypercall_gpa;
1464
1465         printk(KERN_DEBUG "kvm: guest trying to enter paravirtual mode\n");
1466         printk(KERN_DEBUG ".... para_state_gpa: %08Lx\n", para_state_gpa);
1467
1468         /*
1469          * Needs to be page aligned:
1470          */
1471         if (para_state_gpa != PAGE_ALIGN(para_state_gpa))
1472                 goto err_gp;
1473
1474         para_state_hpa = gpa_to_hpa(vcpu, para_state_gpa);
1475         printk(KERN_DEBUG ".... para_state_hpa: %08Lx\n", para_state_hpa);
1476         if (is_error_hpa(para_state_hpa))
1477                 goto err_gp;
1478
1479         mark_page_dirty(vcpu->kvm, para_state_gpa >> PAGE_SHIFT);
1480         para_state_page = pfn_to_page(para_state_hpa >> PAGE_SHIFT);
1481         para_state = kmap_atomic(para_state_page, KM_USER0);
1482
1483         printk(KERN_DEBUG "....  guest version: %d\n", para_state->guest_version);
1484         printk(KERN_DEBUG "....           size: %d\n", para_state->size);
1485
1486         para_state->host_version = KVM_PARA_API_VERSION;
1487         /*
1488          * We cannot support guests that try to register themselves
1489          * with a newer API version than the host supports:
1490          */
1491         if (para_state->guest_version > KVM_PARA_API_VERSION) {
1492                 para_state->ret = -KVM_EINVAL;
1493                 goto err_kunmap_skip;
1494         }
1495
1496         hypercall_gpa = para_state->hypercall_gpa;
1497         hypercall_hpa = gpa_to_hpa(vcpu, hypercall_gpa);
1498         printk(KERN_DEBUG ".... hypercall_hpa: %08Lx\n", hypercall_hpa);
1499         if (is_error_hpa(hypercall_hpa)) {
1500                 para_state->ret = -KVM_EINVAL;
1501                 goto err_kunmap_skip;
1502         }
1503
1504         printk(KERN_DEBUG "kvm: para guest successfully registered.\n");
1505         vcpu->para_state_page = para_state_page;
1506         vcpu->para_state_gpa = para_state_gpa;
1507         vcpu->hypercall_gpa = hypercall_gpa;
1508
1509         mark_page_dirty(vcpu->kvm, hypercall_gpa >> PAGE_SHIFT);
1510         hypercall = kmap_atomic(pfn_to_page(hypercall_hpa >> PAGE_SHIFT),
1511                                 KM_USER1) + (hypercall_hpa & ~PAGE_MASK);
1512         kvm_arch_ops->patch_hypercall(vcpu, hypercall);
1513         kunmap_atomic(hypercall, KM_USER1);
1514
1515         para_state->ret = 0;
1516 err_kunmap_skip:
1517         kunmap_atomic(para_state, KM_USER0);
1518         return 0;
1519 err_gp:
1520         return 1;
1521 }
1522
1523 int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
1524 {
1525         u64 data;
1526
1527         switch (msr) {
1528         case 0xc0010010: /* SYSCFG */
1529         case 0xc0010015: /* HWCR */
1530         case MSR_IA32_PLATFORM_ID:
1531         case MSR_IA32_P5_MC_ADDR:
1532         case MSR_IA32_P5_MC_TYPE:
1533         case MSR_IA32_MC0_CTL:
1534         case MSR_IA32_MCG_STATUS:
1535         case MSR_IA32_MCG_CAP:
1536         case MSR_IA32_MC0_MISC:
1537         case MSR_IA32_MC0_MISC+4:
1538         case MSR_IA32_MC0_MISC+8:
1539         case MSR_IA32_MC0_MISC+12:
1540         case MSR_IA32_MC0_MISC+16:
1541         case MSR_IA32_UCODE_REV:
1542         case MSR_IA32_PERF_STATUS:
1543         case MSR_IA32_EBL_CR_POWERON:
1544                 /* MTRR registers */
1545         case 0xfe:
1546         case 0x200 ... 0x2ff:
1547                 data = 0;
1548                 break;
1549         case 0xcd: /* fsb frequency */
1550                 data = 3;
1551                 break;
1552         case MSR_IA32_APICBASE:
1553                 data = vcpu->apic_base;
1554                 break;
1555         case MSR_IA32_MISC_ENABLE:
1556                 data = vcpu->ia32_misc_enable_msr;
1557                 break;
1558 #ifdef CONFIG_X86_64
1559         case MSR_EFER:
1560                 data = vcpu->shadow_efer;
1561                 break;
1562 #endif
1563         default:
1564                 printk(KERN_ERR "kvm: unhandled rdmsr: 0x%x\n", msr);
1565                 return 1;
1566         }
1567         *pdata = data;
1568         return 0;
1569 }
1570 EXPORT_SYMBOL_GPL(kvm_get_msr_common);
1571
1572 /*
1573  * Reads an msr value (of 'msr_index') into 'pdata'.
1574  * Returns 0 on success, non-0 otherwise.
1575  * Assumes vcpu_load() was already called.
1576  */
1577 static int get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
1578 {
1579         return kvm_arch_ops->get_msr(vcpu, msr_index, pdata);
1580 }
1581
1582 #ifdef CONFIG_X86_64
1583
1584 static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
1585 {
1586         if (efer & EFER_RESERVED_BITS) {
1587                 printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n",
1588                        efer);
1589                 inject_gp(vcpu);
1590                 return;
1591         }
1592
1593         if (is_paging(vcpu)
1594             && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) {
1595                 printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n");
1596                 inject_gp(vcpu);
1597                 return;
1598         }
1599
1600         kvm_arch_ops->set_efer(vcpu, efer);
1601
1602         efer &= ~EFER_LMA;
1603         efer |= vcpu->shadow_efer & EFER_LMA;
1604
1605         vcpu->shadow_efer = efer;
1606 }
1607
1608 #endif
1609
1610 int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
1611 {
1612         switch (msr) {
1613 #ifdef CONFIG_X86_64
1614         case MSR_EFER:
1615                 set_efer(vcpu, data);
1616                 break;
1617 #endif
1618         case MSR_IA32_MC0_STATUS:
1619                 printk(KERN_WARNING "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n",
1620                        __FUNCTION__, data);
1621                 break;
1622         case MSR_IA32_MCG_STATUS:
1623                 printk(KERN_WARNING "%s: MSR_IA32_MCG_STATUS 0x%llx, nop\n",
1624                         __FUNCTION__, data);
1625                 break;
1626         case MSR_IA32_UCODE_REV:
1627         case MSR_IA32_UCODE_WRITE:
1628         case 0x200 ... 0x2ff: /* MTRRs */
1629                 break;
1630         case MSR_IA32_APICBASE:
1631                 vcpu->apic_base = data;
1632                 break;
1633         case MSR_IA32_MISC_ENABLE:
1634                 vcpu->ia32_misc_enable_msr = data;
1635                 break;
1636         /*
1637          * This is the 'probe whether the host is KVM' logic:
1638          */
1639         case MSR_KVM_API_MAGIC:
1640                 return vcpu_register_para(vcpu, data);
1641
1642         default:
1643                 printk(KERN_ERR "kvm: unhandled wrmsr: 0x%x\n", msr);
1644                 return 1;
1645         }
1646         return 0;
1647 }
1648 EXPORT_SYMBOL_GPL(kvm_set_msr_common);
1649
1650 /*
1651  * Writes msr value into into the appropriate "register".
1652  * Returns 0 on success, non-0 otherwise.
1653  * Assumes vcpu_load() was already called.
1654  */
1655 static int set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
1656 {
1657         return kvm_arch_ops->set_msr(vcpu, msr_index, data);
1658 }
1659
1660 void kvm_resched(struct kvm_vcpu *vcpu)
1661 {
1662         if (!need_resched())
1663                 return;
1664         vcpu_put(vcpu);
1665         cond_resched();
1666         vcpu_load(vcpu);
1667 }
1668 EXPORT_SYMBOL_GPL(kvm_resched);
1669
1670 void load_msrs(struct vmx_msr_entry *e, int n)
1671 {
1672         int i;
1673
1674         for (i = 0; i < n; ++i)
1675                 wrmsrl(e[i].index, e[i].data);
1676 }
1677 EXPORT_SYMBOL_GPL(load_msrs);
1678
1679 void save_msrs(struct vmx_msr_entry *e, int n)
1680 {
1681         int i;
1682
1683         for (i = 0; i < n; ++i)
1684                 rdmsrl(e[i].index, e[i].data);
1685 }
1686 EXPORT_SYMBOL_GPL(save_msrs);
1687
1688 void kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
1689 {
1690         int i;
1691         u32 function;
1692         struct kvm_cpuid_entry *e, *best;
1693
1694         kvm_arch_ops->cache_regs(vcpu);
1695         function = vcpu->regs[VCPU_REGS_RAX];
1696         vcpu->regs[VCPU_REGS_RAX] = 0;
1697         vcpu->regs[VCPU_REGS_RBX] = 0;
1698         vcpu->regs[VCPU_REGS_RCX] = 0;
1699         vcpu->regs[VCPU_REGS_RDX] = 0;
1700         best = NULL;
1701         for (i = 0; i < vcpu->cpuid_nent; ++i) {
1702                 e = &vcpu->cpuid_entries[i];
1703                 if (e->function == function) {
1704                         best = e;
1705                         break;
1706                 }
1707                 /*
1708                  * Both basic or both extended?
1709                  */
1710                 if (((e->function ^ function) & 0x80000000) == 0)
1711                         if (!best || e->function > best->function)
1712                                 best = e;
1713         }
1714         if (best) {
1715                 vcpu->regs[VCPU_REGS_RAX] = best->eax;
1716                 vcpu->regs[VCPU_REGS_RBX] = best->ebx;
1717                 vcpu->regs[VCPU_REGS_RCX] = best->ecx;
1718                 vcpu->regs[VCPU_REGS_RDX] = best->edx;
1719         }
1720         kvm_arch_ops->decache_regs(vcpu);
1721         kvm_arch_ops->skip_emulated_instruction(vcpu);
1722 }
1723 EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);
1724
1725 static int pio_copy_data(struct kvm_vcpu *vcpu)
1726 {
1727         void *p = vcpu->pio_data;
1728         void *q;
1729         unsigned bytes;
1730         int nr_pages = vcpu->pio.guest_pages[1] ? 2 : 1;
1731
1732         kvm_arch_ops->vcpu_put(vcpu);
1733         q = vmap(vcpu->pio.guest_pages, nr_pages, VM_READ|VM_WRITE,
1734                  PAGE_KERNEL);
1735         if (!q) {
1736                 kvm_arch_ops->vcpu_load(vcpu);
1737                 free_pio_guest_pages(vcpu);
1738                 return -ENOMEM;
1739         }
1740         q += vcpu->pio.guest_page_offset;
1741         bytes = vcpu->pio.size * vcpu->pio.cur_count;
1742         if (vcpu->pio.in)
1743                 memcpy(q, p, bytes);
1744         else
1745                 memcpy(p, q, bytes);
1746         q -= vcpu->pio.guest_page_offset;
1747         vunmap(q);
1748         kvm_arch_ops->vcpu_load(vcpu);
1749         free_pio_guest_pages(vcpu);
1750         return 0;
1751 }
1752
1753 static int complete_pio(struct kvm_vcpu *vcpu)
1754 {
1755         struct kvm_pio_request *io = &vcpu->pio;
1756         long delta;
1757         int r;
1758
1759         kvm_arch_ops->cache_regs(vcpu);
1760
1761         if (!io->string) {
1762                 if (io->in)
1763                         memcpy(&vcpu->regs[VCPU_REGS_RAX], vcpu->pio_data,
1764                                io->size);
1765         } else {
1766                 if (io->in) {
1767                         r = pio_copy_data(vcpu);
1768                         if (r) {
1769                                 kvm_arch_ops->cache_regs(vcpu);
1770                                 return r;
1771                         }
1772                 }
1773
1774                 delta = 1;
1775                 if (io->rep) {
1776                         delta *= io->cur_count;
1777                         /*
1778                          * The size of the register should really depend on
1779                          * current address size.
1780                          */
1781                         vcpu->regs[VCPU_REGS_RCX] -= delta;
1782                 }
1783                 if (io->down)
1784                         delta = -delta;
1785                 delta *= io->size;
1786                 if (io->in)
1787                         vcpu->regs[VCPU_REGS_RDI] += delta;
1788                 else
1789                         vcpu->regs[VCPU_REGS_RSI] += delta;
1790         }
1791
1792         kvm_arch_ops->decache_regs(vcpu);
1793
1794         io->count -= io->cur_count;
1795         io->cur_count = 0;
1796
1797         if (!io->count)
1798                 kvm_arch_ops->skip_emulated_instruction(vcpu);
1799         return 0;
1800 }
1801
1802 int kvm_setup_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
1803                   int size, unsigned long count, int string, int down,
1804                   gva_t address, int rep, unsigned port)
1805 {
1806         unsigned now, in_page;
1807         int i;
1808         int nr_pages = 1;
1809         struct page *page;
1810
1811         vcpu->run->exit_reason = KVM_EXIT_IO;
1812         vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
1813         vcpu->run->io.size = size;
1814         vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
1815         vcpu->run->io.count = count;
1816         vcpu->run->io.port = port;
1817         vcpu->pio.count = count;
1818         vcpu->pio.cur_count = count;
1819         vcpu->pio.size = size;
1820         vcpu->pio.in = in;
1821         vcpu->pio.string = string;
1822         vcpu->pio.down = down;
1823         vcpu->pio.guest_page_offset = offset_in_page(address);
1824         vcpu->pio.rep = rep;
1825
1826         if (!string) {
1827                 kvm_arch_ops->cache_regs(vcpu);
1828                 memcpy(vcpu->pio_data, &vcpu->regs[VCPU_REGS_RAX], 4);
1829                 kvm_arch_ops->decache_regs(vcpu);
1830                 return 0;
1831         }
1832
1833         if (!count) {
1834                 kvm_arch_ops->skip_emulated_instruction(vcpu);
1835                 return 1;
1836         }
1837
1838         now = min(count, PAGE_SIZE / size);
1839
1840         if (!down)
1841                 in_page = PAGE_SIZE - offset_in_page(address);
1842         else
1843                 in_page = offset_in_page(address) + size;
1844         now = min(count, (unsigned long)in_page / size);
1845         if (!now) {
1846                 /*
1847                  * String I/O straddles page boundary.  Pin two guest pages
1848                  * so that we satisfy atomicity constraints.  Do just one
1849                  * transaction to avoid complexity.
1850                  */
1851                 nr_pages = 2;
1852                 now = 1;
1853         }
1854         if (down) {
1855                 /*
1856                  * String I/O in reverse.  Yuck.  Kill the guest, fix later.
1857                  */
1858                 printk(KERN_ERR "kvm: guest string pio down\n");
1859                 inject_gp(vcpu);
1860                 return 1;
1861         }
1862         vcpu->run->io.count = now;
1863         vcpu->pio.cur_count = now;
1864
1865         for (i = 0; i < nr_pages; ++i) {
1866                 spin_lock(&vcpu->kvm->lock);
1867                 page = gva_to_page(vcpu, address + i * PAGE_SIZE);
1868                 if (page)
1869                         get_page(page);
1870                 vcpu->pio.guest_pages[i] = page;
1871                 spin_unlock(&vcpu->kvm->lock);
1872                 if (!page) {
1873                         inject_gp(vcpu);
1874                         free_pio_guest_pages(vcpu);
1875                         return 1;
1876                 }
1877         }
1878
1879         if (!vcpu->pio.in)
1880                 return pio_copy_data(vcpu);
1881         return 0;
1882 }
1883 EXPORT_SYMBOL_GPL(kvm_setup_pio);
1884
1885 static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1886 {
1887         int r;
1888         sigset_t sigsaved;
1889
1890         vcpu_load(vcpu);
1891
1892         if (vcpu->sigset_active)
1893                 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1894
1895         /* re-sync apic's tpr */
1896         vcpu->cr8 = kvm_run->cr8;
1897
1898         if (vcpu->pio.cur_count) {
1899                 r = complete_pio(vcpu);
1900                 if (r)
1901                         goto out;
1902         }
1903
1904         if (vcpu->mmio_needed) {
1905                 memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
1906                 vcpu->mmio_read_completed = 1;
1907                 vcpu->mmio_needed = 0;
1908                 r = emulate_instruction(vcpu, kvm_run,
1909                                         vcpu->mmio_fault_cr2, 0);
1910                 if (r == EMULATE_DO_MMIO) {
1911                         /*
1912                          * Read-modify-write.  Back to userspace.
1913                          */
1914                         kvm_run->exit_reason = KVM_EXIT_MMIO;
1915                         r = 0;
1916                         goto out;
1917                 }
1918         }
1919
1920         if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) {
1921                 kvm_arch_ops->cache_regs(vcpu);
1922                 vcpu->regs[VCPU_REGS_RAX] = kvm_run->hypercall.ret;
1923                 kvm_arch_ops->decache_regs(vcpu);
1924         }
1925
1926         r = kvm_arch_ops->run(vcpu, kvm_run);
1927
1928 out:
1929         if (vcpu->sigset_active)
1930                 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1931
1932         vcpu_put(vcpu);
1933         return r;
1934 }
1935
1936 static int kvm_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu,
1937                                    struct kvm_regs *regs)
1938 {
1939         vcpu_load(vcpu);
1940
1941         kvm_arch_ops->cache_regs(vcpu);
1942
1943         regs->rax = vcpu->regs[VCPU_REGS_RAX];
1944         regs->rbx = vcpu->regs[VCPU_REGS_RBX];
1945         regs->rcx = vcpu->regs[VCPU_REGS_RCX];
1946         regs->rdx = vcpu->regs[VCPU_REGS_RDX];
1947         regs->rsi = vcpu->regs[VCPU_REGS_RSI];
1948         regs->rdi = vcpu->regs[VCPU_REGS_RDI];
1949         regs->rsp = vcpu->regs[VCPU_REGS_RSP];
1950         regs->rbp = vcpu->regs[VCPU_REGS_RBP];
1951 #ifdef CONFIG_X86_64
1952         regs->r8 = vcpu->regs[VCPU_REGS_R8];
1953         regs->r9 = vcpu->regs[VCPU_REGS_R9];
1954         regs->r10 = vcpu->regs[VCPU_REGS_R10];
1955         regs->r11 = vcpu->regs[VCPU_REGS_R11];
1956         regs->r12 = vcpu->regs[VCPU_REGS_R12];
1957         regs->r13 = vcpu->regs[VCPU_REGS_R13];
1958         regs->r14 = vcpu->regs[VCPU_REGS_R14];
1959         regs->r15 = vcpu->regs[VCPU_REGS_R15];
1960 #endif
1961
1962         regs->rip = vcpu->rip;
1963         regs->rflags = kvm_arch_ops->get_rflags(vcpu);
1964
1965         /*
1966          * Don't leak debug flags in case they were set for guest debugging
1967          */
1968         if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep)
1969                 regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
1970
1971         vcpu_put(vcpu);
1972
1973         return 0;
1974 }
1975
1976 static int kvm_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu,
1977                                    struct kvm_regs *regs)
1978 {
1979         vcpu_load(vcpu);
1980
1981         vcpu->regs[VCPU_REGS_RAX] = regs->rax;
1982         vcpu->regs[VCPU_REGS_RBX] = regs->rbx;
1983         vcpu->regs[VCPU_REGS_RCX] = regs->rcx;
1984         vcpu->regs[VCPU_REGS_RDX] = regs->rdx;
1985         vcpu->regs[VCPU_REGS_RSI] = regs->rsi;
1986         vcpu->regs[VCPU_REGS_RDI] = regs->rdi;
1987         vcpu->regs[VCPU_REGS_RSP] = regs->rsp;
1988         vcpu->regs[VCPU_REGS_RBP] = regs->rbp;
1989 #ifdef CONFIG_X86_64
1990         vcpu->regs[VCPU_REGS_R8] = regs->r8;
1991         vcpu->regs[VCPU_REGS_R9] = regs->r9;
1992         vcpu->regs[VCPU_REGS_R10] = regs->r10;
1993         vcpu->regs[VCPU_REGS_R11] = regs->r11;
1994         vcpu->regs[VCPU_REGS_R12] = regs->r12;
1995         vcpu->regs[VCPU_REGS_R13] = regs->r13;
1996         vcpu->regs[VCPU_REGS_R14] = regs->r14;
1997         vcpu->regs[VCPU_REGS_R15] = regs->r15;
1998 #endif
1999
2000         vcpu->rip = regs->rip;
2001         kvm_arch_ops->set_rflags(vcpu, regs->rflags);
2002
2003         kvm_arch_ops->decache_regs(vcpu);
2004
2005         vcpu_put(vcpu);
2006
2007         return 0;
2008 }
2009
2010 static void get_segment(struct kvm_vcpu *vcpu,
2011                         struct kvm_segment *var, int seg)
2012 {
2013         return kvm_arch_ops->get_segment(vcpu, var, seg);
2014 }
2015
2016 static int kvm_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2017                                     struct kvm_sregs *sregs)
2018 {
2019         struct descriptor_table dt;
2020
2021         vcpu_load(vcpu);
2022
2023         get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
2024         get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
2025         get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
2026         get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
2027         get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
2028         get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
2029
2030         get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
2031         get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
2032
2033         kvm_arch_ops->get_idt(vcpu, &dt);
2034         sregs->idt.limit = dt.limit;
2035         sregs->idt.base = dt.base;
2036         kvm_arch_ops->get_gdt(vcpu, &dt);
2037         sregs->gdt.limit = dt.limit;
2038         sregs->gdt.base = dt.base;
2039
2040         kvm_arch_ops->decache_cr4_guest_bits(vcpu);
2041         sregs->cr0 = vcpu->cr0;
2042         sregs->cr2 = vcpu->cr2;
2043         sregs->cr3 = vcpu->cr3;
2044         sregs->cr4 = vcpu->cr4;
2045         sregs->cr8 = vcpu->cr8;
2046         sregs->efer = vcpu->shadow_efer;
2047         sregs->apic_base = vcpu->apic_base;
2048
2049         memcpy(sregs->interrupt_bitmap, vcpu->irq_pending,
2050                sizeof sregs->interrupt_bitmap);
2051
2052         vcpu_put(vcpu);
2053
2054         return 0;
2055 }
2056
2057 static void set_segment(struct kvm_vcpu *vcpu,
2058                         struct kvm_segment *var, int seg)
2059 {
2060         return kvm_arch_ops->set_segment(vcpu, var, seg);
2061 }
2062
2063 static int kvm_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2064                                     struct kvm_sregs *sregs)
2065 {
2066         int mmu_reset_needed = 0;
2067         int i;
2068         struct descriptor_table dt;
2069
2070         vcpu_load(vcpu);
2071
2072         dt.limit = sregs->idt.limit;
2073         dt.base = sregs->idt.base;
2074         kvm_arch_ops->set_idt(vcpu, &dt);
2075         dt.limit = sregs->gdt.limit;
2076         dt.base = sregs->gdt.base;
2077         kvm_arch_ops->set_gdt(vcpu, &dt);
2078
2079         vcpu->cr2 = sregs->cr2;
2080         mmu_reset_needed |= vcpu->cr3 != sregs->cr3;
2081         vcpu->cr3 = sregs->cr3;
2082
2083         vcpu->cr8 = sregs->cr8;
2084
2085         mmu_reset_needed |= vcpu->shadow_efer != sregs->efer;
2086 #ifdef CONFIG_X86_64
2087         kvm_arch_ops->set_efer(vcpu, sregs->efer);
2088 #endif
2089         vcpu->apic_base = sregs->apic_base;
2090
2091         kvm_arch_ops->decache_cr4_guest_bits(vcpu);
2092
2093         mmu_reset_needed |= vcpu->cr0 != sregs->cr0;
2094         kvm_arch_ops->set_cr0(vcpu, sregs->cr0);
2095
2096         mmu_reset_needed |= vcpu->cr4 != sregs->cr4;
2097         kvm_arch_ops->set_cr4(vcpu, sregs->cr4);
2098         if (!is_long_mode(vcpu) && is_pae(vcpu))
2099                 load_pdptrs(vcpu, vcpu->cr3);
2100
2101         if (mmu_reset_needed)
2102                 kvm_mmu_reset_context(vcpu);
2103
2104         memcpy(vcpu->irq_pending, sregs->interrupt_bitmap,
2105                sizeof vcpu->irq_pending);
2106         vcpu->irq_summary = 0;
2107         for (i = 0; i < NR_IRQ_WORDS; ++i)
2108                 if (vcpu->irq_pending[i])
2109                         __set_bit(i, &vcpu->irq_summary);
2110
2111         set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
2112         set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
2113         set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
2114         set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
2115         set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
2116         set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
2117
2118         set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
2119         set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
2120
2121         vcpu_put(vcpu);
2122
2123         return 0;
2124 }
2125
2126 /*
2127  * List of msr numbers which we expose to userspace through KVM_GET_MSRS
2128  * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
2129  *
2130  * This list is modified at module load time to reflect the
2131  * capabilities of the host cpu.
2132  */
2133 static u32 msrs_to_save[] = {
2134         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
2135         MSR_K6_STAR,
2136 #ifdef CONFIG_X86_64
2137         MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
2138 #endif
2139         MSR_IA32_TIME_STAMP_COUNTER,
2140 };
2141
2142 static unsigned num_msrs_to_save;
2143
2144 static u32 emulated_msrs[] = {
2145         MSR_IA32_MISC_ENABLE,
2146 };
2147
2148 static __init void kvm_init_msr_list(void)
2149 {
2150         u32 dummy[2];
2151         unsigned i, j;
2152
2153         for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) {
2154                 if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
2155                         continue;
2156                 if (j < i)
2157                         msrs_to_save[j] = msrs_to_save[i];
2158                 j++;
2159         }
2160         num_msrs_to_save = j;
2161 }
2162
2163 /*
2164  * Adapt set_msr() to msr_io()'s calling convention
2165  */
2166 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
2167 {
2168         return set_msr(vcpu, index, *data);
2169 }
2170
2171 /*
2172  * Read or write a bunch of msrs. All parameters are kernel addresses.
2173  *
2174  * @return number of msrs set successfully.
2175  */
2176 static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
2177                     struct kvm_msr_entry *entries,
2178                     int (*do_msr)(struct kvm_vcpu *vcpu,
2179                                   unsigned index, u64 *data))
2180 {
2181         int i;
2182
2183         vcpu_load(vcpu);
2184
2185         for (i = 0; i < msrs->nmsrs; ++i)
2186                 if (do_msr(vcpu, entries[i].index, &entries[i].data))
2187                         break;
2188
2189         vcpu_put(vcpu);
2190
2191         return i;
2192 }
2193
2194 /*
2195  * Read or write a bunch of msrs. Parameters are user addresses.
2196  *
2197  * @return number of msrs set successfully.
2198  */
2199 static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
2200                   int (*do_msr)(struct kvm_vcpu *vcpu,
2201                                 unsigned index, u64 *data),
2202                   int writeback)
2203 {
2204         struct kvm_msrs msrs;
2205         struct kvm_msr_entry *entries;
2206         int r, n;
2207         unsigned size;
2208
2209         r = -EFAULT;
2210         if (copy_from_user(&msrs, user_msrs, sizeof msrs))
2211                 goto out;
2212
2213         r = -E2BIG;
2214         if (msrs.nmsrs >= MAX_IO_MSRS)
2215                 goto out;
2216
2217         r = -ENOMEM;
2218         size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
2219         entries = vmalloc(size);
2220         if (!entries)
2221                 goto out;
2222
2223         r = -EFAULT;
2224         if (copy_from_user(entries, user_msrs->entries, size))
2225                 goto out_free;
2226
2227         r = n = __msr_io(vcpu, &msrs, entries, do_msr);
2228         if (r < 0)
2229                 goto out_free;
2230
2231         r = -EFAULT;
2232         if (writeback && copy_to_user(user_msrs->entries, entries, size))
2233                 goto out_free;
2234
2235         r = n;
2236
2237 out_free:
2238         vfree(entries);
2239 out:
2240         return r;
2241 }
2242
2243 /*
2244  * Translate a guest virtual address to a guest physical address.
2245  */
2246 static int kvm_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2247                                     struct kvm_translation *tr)
2248 {
2249         unsigned long vaddr = tr->linear_address;
2250         gpa_t gpa;
2251
2252         vcpu_load(vcpu);
2253         spin_lock(&vcpu->kvm->lock);
2254         gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr);
2255         tr->physical_address = gpa;
2256         tr->valid = gpa != UNMAPPED_GVA;
2257         tr->writeable = 1;
2258         tr->usermode = 0;
2259         spin_unlock(&vcpu->kvm->lock);
2260         vcpu_put(vcpu);
2261
2262         return 0;
2263 }
2264
2265 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
2266                                     struct kvm_interrupt *irq)
2267 {
2268         if (irq->irq < 0 || irq->irq >= 256)
2269                 return -EINVAL;
2270         vcpu_load(vcpu);
2271
2272         set_bit(irq->irq, vcpu->irq_pending);
2273         set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary);
2274
2275         vcpu_put(vcpu);
2276
2277         return 0;
2278 }
2279
2280 static int kvm_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu,
2281                                       struct kvm_debug_guest *dbg)
2282 {
2283         int r;
2284
2285         vcpu_load(vcpu);
2286
2287         r = kvm_arch_ops->set_guest_debug(vcpu, dbg);
2288
2289         vcpu_put(vcpu);
2290
2291         return r;
2292 }
2293
2294 static struct page *kvm_vcpu_nopage(struct vm_area_struct *vma,
2295                                     unsigned long address,
2296                                     int *type)
2297 {
2298         struct kvm_vcpu *vcpu = vma->vm_file->private_data;
2299         unsigned long pgoff;
2300         struct page *page;
2301
2302         *type = VM_FAULT_MINOR;
2303         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2304         if (pgoff == 0)
2305                 page = virt_to_page(vcpu->run);
2306         else if (pgoff == KVM_PIO_PAGE_OFFSET)
2307                 page = virt_to_page(vcpu->pio_data);
2308         else
2309                 return NOPAGE_SIGBUS;
2310         get_page(page);
2311         return page;
2312 }
2313
2314 static struct vm_operations_struct kvm_vcpu_vm_ops = {
2315         .nopage = kvm_vcpu_nopage,
2316 };
2317
2318 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
2319 {
2320         vma->vm_ops = &kvm_vcpu_vm_ops;
2321         return 0;
2322 }
2323
2324 static int kvm_vcpu_release(struct inode *inode, struct file *filp)
2325 {
2326         struct kvm_vcpu *vcpu = filp->private_data;
2327
2328         fput(vcpu->kvm->filp);
2329         return 0;
2330 }
2331
2332 static struct file_operations kvm_vcpu_fops = {
2333         .release        = kvm_vcpu_release,
2334         .unlocked_ioctl = kvm_vcpu_ioctl,
2335         .compat_ioctl   = kvm_vcpu_ioctl,
2336         .mmap           = kvm_vcpu_mmap,
2337 };
2338
2339 /*
2340  * Allocates an inode for the vcpu.
2341  */
2342 static int create_vcpu_fd(struct kvm_vcpu *vcpu)
2343 {
2344         int fd, r;
2345         struct inode *inode;
2346         struct file *file;
2347
2348         atomic_inc(&vcpu->kvm->filp->f_count);
2349         inode = kvmfs_inode(&kvm_vcpu_fops);
2350         if (IS_ERR(inode)) {
2351                 r = PTR_ERR(inode);
2352                 goto out1;
2353         }
2354
2355         file = kvmfs_file(inode, vcpu);
2356         if (IS_ERR(file)) {
2357                 r = PTR_ERR(file);
2358                 goto out2;
2359         }
2360
2361         r = get_unused_fd();
2362         if (r < 0)
2363                 goto out3;
2364         fd = r;
2365         fd_install(fd, file);
2366
2367         return fd;
2368
2369 out3:
2370         fput(file);
2371 out2:
2372         iput(inode);
2373 out1:
2374         fput(vcpu->kvm->filp);
2375         return r;
2376 }
2377
2378 /*
2379  * Creates some virtual cpus.  Good luck creating more than one.
2380  */
2381 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
2382 {
2383         int r;
2384         struct kvm_vcpu *vcpu;
2385         struct page *page;
2386
2387         r = -EINVAL;
2388         if (!valid_vcpu(n))
2389                 goto out;
2390
2391         vcpu = &kvm->vcpus[n];
2392
2393         mutex_lock(&vcpu->mutex);
2394
2395         if (vcpu->vmcs) {
2396                 mutex_unlock(&vcpu->mutex);
2397                 return -EEXIST;
2398         }
2399
2400         page = alloc_page(GFP_KERNEL | __GFP_ZERO);
2401         r = -ENOMEM;
2402         if (!page)
2403                 goto out_unlock;
2404         vcpu->run = page_address(page);
2405
2406         page = alloc_page(GFP_KERNEL | __GFP_ZERO);
2407         r = -ENOMEM;
2408         if (!page)
2409                 goto out_free_run;
2410         vcpu->pio_data = page_address(page);
2411
2412         vcpu->host_fx_image = (char*)ALIGN((hva_t)vcpu->fx_buf,
2413                                            FX_IMAGE_ALIGN);
2414         vcpu->guest_fx_image = vcpu->host_fx_image + FX_IMAGE_SIZE;
2415         vcpu->cr0 = 0x10;
2416
2417         r = kvm_arch_ops->vcpu_create(vcpu);
2418         if (r < 0)
2419                 goto out_free_vcpus;
2420
2421         r = kvm_mmu_create(vcpu);
2422         if (r < 0)
2423                 goto out_free_vcpus;
2424
2425         kvm_arch_ops->vcpu_load(vcpu);
2426         r = kvm_mmu_setup(vcpu);
2427         if (r >= 0)
2428                 r = kvm_arch_ops->vcpu_setup(vcpu);
2429         vcpu_put(vcpu);
2430
2431         if (r < 0)
2432                 goto out_free_vcpus;
2433
2434         r = create_vcpu_fd(vcpu);
2435         if (r < 0)
2436                 goto out_free_vcpus;
2437
2438         spin_lock(&kvm_lock);
2439         if (n >= kvm->nvcpus)
2440                 kvm->nvcpus = n + 1;
2441         spin_unlock(&kvm_lock);
2442
2443         return r;
2444
2445 out_free_vcpus:
2446         kvm_free_vcpu(vcpu);
2447 out_free_run:
2448         free_page((unsigned long)vcpu->run);
2449         vcpu->run = NULL;
2450 out_unlock:
2451         mutex_unlock(&vcpu->mutex);
2452 out:
2453         return r;
2454 }
2455
2456 static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu)
2457 {
2458         u64 efer;
2459         int i;
2460         struct kvm_cpuid_entry *e, *entry;
2461
2462         rdmsrl(MSR_EFER, efer);
2463         entry = NULL;
2464         for (i = 0; i < vcpu->cpuid_nent; ++i) {
2465                 e = &vcpu->cpuid_entries[i];
2466                 if (e->function == 0x80000001) {
2467                         entry = e;
2468                         break;
2469                 }
2470         }
2471         if (entry && (entry->edx & EFER_NX) && !(efer & EFER_NX)) {
2472                 entry->edx &= ~(1 << 20);
2473                 printk(KERN_INFO ": guest NX capability removed\n");
2474         }
2475 }
2476
2477 static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
2478                                     struct kvm_cpuid *cpuid,
2479                                     struct kvm_cpuid_entry __user *entries)
2480 {
2481         int r;
2482
2483         r = -E2BIG;
2484         if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
2485                 goto out;
2486         r = -EFAULT;
2487         if (copy_from_user(&vcpu->cpuid_entries, entries,
2488                            cpuid->nent * sizeof(struct kvm_cpuid_entry)))
2489                 goto out;
2490         vcpu->cpuid_nent = cpuid->nent;
2491         cpuid_fix_nx_cap(vcpu);
2492         return 0;
2493
2494 out:
2495         return r;
2496 }
2497
2498 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
2499 {
2500         if (sigset) {
2501                 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
2502                 vcpu->sigset_active = 1;
2503                 vcpu->sigset = *sigset;
2504         } else
2505                 vcpu->sigset_active = 0;
2506         return 0;
2507 }
2508
2509 /*
2510  * fxsave fpu state.  Taken from x86_64/processor.h.  To be killed when
2511  * we have asm/x86/processor.h
2512  */
2513 struct fxsave {
2514         u16     cwd;
2515         u16     swd;
2516         u16     twd;
2517         u16     fop;
2518         u64     rip;
2519         u64     rdp;
2520         u32     mxcsr;
2521         u32     mxcsr_mask;
2522         u32     st_space[32];   /* 8*16 bytes for each FP-reg = 128 bytes */
2523 #ifdef CONFIG_X86_64
2524         u32     xmm_space[64];  /* 16*16 bytes for each XMM-reg = 256 bytes */
2525 #else
2526         u32     xmm_space[32];  /* 8*16 bytes for each XMM-reg = 128 bytes */
2527 #endif
2528 };
2529
2530 static int kvm_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2531 {
2532         struct fxsave *fxsave = (struct fxsave *)vcpu->guest_fx_image;
2533
2534         vcpu_load(vcpu);
2535
2536         memcpy(fpu->fpr, fxsave->st_space, 128);
2537         fpu->fcw = fxsave->cwd;
2538         fpu->fsw = fxsave->swd;
2539         fpu->ftwx = fxsave->twd;
2540         fpu->last_opcode = fxsave->fop;
2541         fpu->last_ip = fxsave->rip;
2542         fpu->last_dp = fxsave->rdp;
2543         memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space);
2544
2545         vcpu_put(vcpu);
2546
2547         return 0;
2548 }
2549
2550 static int kvm_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2551 {
2552         struct fxsave *fxsave = (struct fxsave *)vcpu->guest_fx_image;
2553
2554         vcpu_load(vcpu);
2555
2556         memcpy(fxsave->st_space, fpu->fpr, 128);
2557         fxsave->cwd = fpu->fcw;
2558         fxsave->swd = fpu->fsw;
2559         fxsave->twd = fpu->ftwx;
2560         fxsave->fop = fpu->last_opcode;
2561         fxsave->rip = fpu->last_ip;
2562         fxsave->rdp = fpu->last_dp;
2563         memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space);
2564
2565         vcpu_put(vcpu);
2566
2567         return 0;
2568 }
2569
2570 static long kvm_vcpu_ioctl(struct file *filp,
2571                            unsigned int ioctl, unsigned long arg)
2572 {
2573         struct kvm_vcpu *vcpu = filp->private_data;
2574         void __user *argp = (void __user *)arg;
2575         int r = -EINVAL;
2576
2577         switch (ioctl) {
2578         case KVM_RUN:
2579                 r = -EINVAL;
2580                 if (arg)
2581                         goto out;
2582                 r = kvm_vcpu_ioctl_run(vcpu, vcpu->run);
2583                 break;
2584         case KVM_GET_REGS: {
2585                 struct kvm_regs kvm_regs;
2586
2587                 memset(&kvm_regs, 0, sizeof kvm_regs);
2588                 r = kvm_vcpu_ioctl_get_regs(vcpu, &kvm_regs);
2589                 if (r)
2590                         goto out;
2591                 r = -EFAULT;
2592                 if (copy_to_user(argp, &kvm_regs, sizeof kvm_regs))
2593                         goto out;
2594                 r = 0;
2595                 break;
2596         }
2597         case KVM_SET_REGS: {
2598                 struct kvm_regs kvm_regs;
2599
2600                 r = -EFAULT;
2601                 if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs))
2602                         goto out;
2603                 r = kvm_vcpu_ioctl_set_regs(vcpu, &kvm_regs);
2604                 if (r)
2605                         goto out;
2606                 r = 0;
2607                 break;
2608         }
2609         case KVM_GET_SREGS: {
2610                 struct kvm_sregs kvm_sregs;
2611
2612                 memset(&kvm_sregs, 0, sizeof kvm_sregs);
2613                 r = kvm_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs);
2614                 if (r)
2615                         goto out;
2616                 r = -EFAULT;
2617                 if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs))
2618                         goto out;
2619                 r = 0;
2620                 break;
2621         }
2622         case KVM_SET_SREGS: {
2623                 struct kvm_sregs kvm_sregs;
2624
2625                 r = -EFAULT;
2626                 if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
2627                         goto out;
2628                 r = kvm_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs);
2629                 if (r)
2630                         goto out;
2631                 r = 0;
2632                 break;
2633         }
2634         case KVM_TRANSLATE: {
2635                 struct kvm_translation tr;
2636
2637                 r = -EFAULT;
2638                 if (copy_from_user(&tr, argp, sizeof tr))
2639                         goto out;
2640                 r = kvm_vcpu_ioctl_translate(vcpu, &tr);
2641                 if (r)
2642                         goto out;
2643                 r = -EFAULT;
2644                 if (copy_to_user(argp, &tr, sizeof tr))
2645                         goto out;
2646                 r = 0;
2647                 break;
2648         }
2649         case KVM_INTERRUPT: {
2650                 struct kvm_interrupt irq;
2651
2652                 r = -EFAULT;
2653                 if (copy_from_user(&irq, argp, sizeof irq))
2654                         goto out;
2655                 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
2656                 if (r)
2657                         goto out;
2658                 r = 0;
2659                 break;
2660         }
2661         case KVM_DEBUG_GUEST: {
2662                 struct kvm_debug_guest dbg;
2663
2664                 r = -EFAULT;
2665                 if (copy_from_user(&dbg, argp, sizeof dbg))
2666                         goto out;
2667                 r = kvm_vcpu_ioctl_debug_guest(vcpu, &dbg);
2668                 if (r)
2669                         goto out;
2670                 r = 0;
2671                 break;
2672         }
2673         case KVM_GET_MSRS:
2674                 r = msr_io(vcpu, argp, get_msr, 1);
2675                 break;
2676         case KVM_SET_MSRS:
2677                 r = msr_io(vcpu, argp, do_set_msr, 0);
2678                 break;
2679         case KVM_SET_CPUID: {
2680                 struct kvm_cpuid __user *cpuid_arg = argp;
2681                 struct kvm_cpuid cpuid;
2682
2683                 r = -EFAULT;
2684                 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
2685                         goto out;
2686                 r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
2687                 if (r)
2688                         goto out;
2689                 break;
2690         }
2691         case KVM_SET_SIGNAL_MASK: {
2692                 struct kvm_signal_mask __user *sigmask_arg = argp;
2693                 struct kvm_signal_mask kvm_sigmask;
2694                 sigset_t sigset, *p;
2695
2696                 p = NULL;
2697                 if (argp) {
2698                         r = -EFAULT;
2699                         if (copy_from_user(&kvm_sigmask, argp,
2700                                            sizeof kvm_sigmask))
2701                                 goto out;
2702                         r = -EINVAL;
2703                         if (kvm_sigmask.len != sizeof sigset)
2704                                 goto out;
2705                         r = -EFAULT;
2706                         if (copy_from_user(&sigset, sigmask_arg->sigset,
2707                                            sizeof sigset))
2708                                 goto out;
2709                         p = &sigset;
2710                 }
2711                 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
2712                 break;
2713         }
2714         case KVM_GET_FPU: {
2715                 struct kvm_fpu fpu;
2716
2717                 memset(&fpu, 0, sizeof fpu);
2718                 r = kvm_vcpu_ioctl_get_fpu(vcpu, &fpu);
2719                 if (r)
2720                         goto out;
2721                 r = -EFAULT;
2722                 if (copy_to_user(argp, &fpu, sizeof fpu))
2723                         goto out;
2724                 r = 0;
2725                 break;
2726         }
2727         case KVM_SET_FPU: {
2728                 struct kvm_fpu fpu;
2729
2730                 r = -EFAULT;
2731                 if (copy_from_user(&fpu, argp, sizeof fpu))
2732                         goto out;
2733                 r = kvm_vcpu_ioctl_set_fpu(vcpu, &fpu);
2734                 if (r)
2735                         goto out;
2736                 r = 0;
2737                 break;
2738         }
2739         default:
2740                 ;
2741         }
2742 out:
2743         return r;
2744 }
2745
2746 static long kvm_vm_ioctl(struct file *filp,
2747                            unsigned int ioctl, unsigned long arg)
2748 {
2749         struct kvm *kvm = filp->private_data;
2750         void __user *argp = (void __user *)arg;
2751         int r = -EINVAL;
2752
2753         switch (ioctl) {
2754         case KVM_CREATE_VCPU:
2755                 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
2756                 if (r < 0)
2757                         goto out;
2758                 break;
2759         case KVM_SET_MEMORY_REGION: {
2760                 struct kvm_memory_region kvm_mem;
2761
2762                 r = -EFAULT;
2763                 if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem))
2764                         goto out;
2765                 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_mem);
2766                 if (r)
2767                         goto out;
2768                 break;
2769         }
2770         case KVM_GET_DIRTY_LOG: {
2771                 struct kvm_dirty_log log;
2772
2773                 r = -EFAULT;
2774                 if (copy_from_user(&log, argp, sizeof log))
2775                         goto out;
2776                 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
2777                 if (r)
2778                         goto out;
2779                 break;
2780         }
2781         case KVM_SET_MEMORY_ALIAS: {
2782                 struct kvm_memory_alias alias;
2783
2784                 r = -EFAULT;
2785                 if (copy_from_user(&alias, argp, sizeof alias))
2786                         goto out;
2787                 r = kvm_vm_ioctl_set_memory_alias(kvm, &alias);
2788                 if (r)
2789                         goto out;
2790                 break;
2791         }
2792         default:
2793                 ;
2794         }
2795 out:
2796         return r;
2797 }
2798
2799 static struct page *kvm_vm_nopage(struct vm_area_struct *vma,
2800                                   unsigned long address,
2801                                   int *type)
2802 {
2803         struct kvm *kvm = vma->vm_file->private_data;
2804         unsigned long pgoff;
2805         struct page *page;
2806
2807         *type = VM_FAULT_MINOR;
2808         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2809         page = gfn_to_page(kvm, pgoff);
2810         if (!page)
2811                 return NOPAGE_SIGBUS;
2812         get_page(page);
2813         return page;
2814 }
2815
2816 static struct vm_operations_struct kvm_vm_vm_ops = {
2817         .nopage = kvm_vm_nopage,
2818 };
2819
2820 static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
2821 {
2822         vma->vm_ops = &kvm_vm_vm_ops;
2823         return 0;
2824 }
2825
2826 static struct file_operations kvm_vm_fops = {
2827         .release        = kvm_vm_release,
2828         .unlocked_ioctl = kvm_vm_ioctl,
2829         .compat_ioctl   = kvm_vm_ioctl,
2830         .mmap           = kvm_vm_mmap,
2831 };
2832
2833 static int kvm_dev_ioctl_create_vm(void)
2834 {
2835         int fd, r;
2836         struct inode *inode;
2837         struct file *file;
2838         struct kvm *kvm;
2839
2840         inode = kvmfs_inode(&kvm_vm_fops);
2841         if (IS_ERR(inode)) {
2842                 r = PTR_ERR(inode);
2843                 goto out1;
2844         }
2845
2846         kvm = kvm_create_vm();
2847         if (IS_ERR(kvm)) {
2848                 r = PTR_ERR(kvm);
2849                 goto out2;
2850         }
2851
2852         file = kvmfs_file(inode, kvm);
2853         if (IS_ERR(file)) {
2854                 r = PTR_ERR(file);
2855                 goto out3;
2856         }
2857         kvm->filp = file;
2858
2859         r = get_unused_fd();
2860         if (r < 0)
2861                 goto out4;
2862         fd = r;
2863         fd_install(fd, file);
2864
2865         return fd;
2866
2867 out4:
2868         fput(file);
2869 out3:
2870         kvm_destroy_vm(kvm);
2871 out2:
2872         iput(inode);
2873 out1:
2874         return r;
2875 }
2876
2877 static long kvm_dev_ioctl(struct file *filp,
2878                           unsigned int ioctl, unsigned long arg)
2879 {
2880         void __user *argp = (void __user *)arg;
2881         long r = -EINVAL;
2882
2883         switch (ioctl) {
2884         case KVM_GET_API_VERSION:
2885                 r = -EINVAL;
2886                 if (arg)
2887                         goto out;
2888                 r = KVM_API_VERSION;
2889                 break;
2890         case KVM_CREATE_VM:
2891                 r = -EINVAL;
2892                 if (arg)
2893                         goto out;
2894                 r = kvm_dev_ioctl_create_vm();
2895                 break;
2896         case KVM_GET_MSR_INDEX_LIST: {
2897                 struct kvm_msr_list __user *user_msr_list = argp;
2898                 struct kvm_msr_list msr_list;
2899                 unsigned n;
2900
2901                 r = -EFAULT;
2902                 if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
2903                         goto out;
2904                 n = msr_list.nmsrs;
2905                 msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
2906                 if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
2907                         goto out;
2908                 r = -E2BIG;
2909                 if (n < num_msrs_to_save)
2910                         goto out;
2911                 r = -EFAULT;
2912                 if (copy_to_user(user_msr_list->indices, &msrs_to_save,
2913                                  num_msrs_to_save * sizeof(u32)))
2914                         goto out;
2915                 if (copy_to_user(user_msr_list->indices
2916                                  + num_msrs_to_save * sizeof(u32),
2917                                  &emulated_msrs,
2918                                  ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
2919                         goto out;
2920                 r = 0;
2921                 break;
2922         }
2923         case KVM_CHECK_EXTENSION:
2924                 /*
2925                  * No extensions defined at present.
2926                  */
2927                 r = 0;
2928                 break;
2929         case KVM_GET_VCPU_MMAP_SIZE:
2930                 r = -EINVAL;
2931                 if (arg)
2932                         goto out;
2933                 r = 2 * PAGE_SIZE;
2934                 break;
2935         default:
2936                 ;
2937         }
2938 out:
2939         return r;
2940 }
2941
2942 static struct file_operations kvm_chardev_ops = {
2943         .open           = kvm_dev_open,
2944         .release        = kvm_dev_release,
2945         .unlocked_ioctl = kvm_dev_ioctl,
2946         .compat_ioctl   = kvm_dev_ioctl,
2947 };
2948
2949 static struct miscdevice kvm_dev = {
2950         KVM_MINOR,
2951         "kvm",
2952         &kvm_chardev_ops,
2953 };
2954
2955 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
2956                        void *v)
2957 {
2958         if (val == SYS_RESTART) {
2959                 /*
2960                  * Some (well, at least mine) BIOSes hang on reboot if
2961                  * in vmx root mode.
2962                  */
2963                 printk(KERN_INFO "kvm: exiting hardware virtualization\n");
2964                 on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
2965         }
2966         return NOTIFY_OK;
2967 }
2968
2969 static struct notifier_block kvm_reboot_notifier = {
2970         .notifier_call = kvm_reboot,
2971         .priority = 0,
2972 };
2973
2974 /*
2975  * Make sure that a cpu that is being hot-unplugged does not have any vcpus
2976  * cached on it.
2977  */
2978 static void decache_vcpus_on_cpu(int cpu)
2979 {
2980         struct kvm *vm;
2981         struct kvm_vcpu *vcpu;
2982         int i;
2983
2984         spin_lock(&kvm_lock);
2985         list_for_each_entry(vm, &vm_list, vm_list)
2986                 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
2987                         vcpu = &vm->vcpus[i];
2988                         /*
2989                          * If the vcpu is locked, then it is running on some
2990                          * other cpu and therefore it is not cached on the
2991                          * cpu in question.
2992                          *
2993                          * If it's not locked, check the last cpu it executed
2994                          * on.
2995                          */
2996                         if (mutex_trylock(&vcpu->mutex)) {
2997                                 if (vcpu->cpu == cpu) {
2998                                         kvm_arch_ops->vcpu_decache(vcpu);
2999                                         vcpu->cpu = -1;
3000                                 }
3001                                 mutex_unlock(&vcpu->mutex);
3002                         }
3003                 }
3004         spin_unlock(&kvm_lock);
3005 }
3006
3007 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
3008                            void *v)
3009 {
3010         int cpu = (long)v;
3011
3012         switch (val) {
3013         case CPU_DOWN_PREPARE:
3014         case CPU_DOWN_PREPARE_FROZEN:
3015         case CPU_UP_CANCELED:
3016         case CPU_UP_CANCELED_FROZEN:
3017                 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
3018                        cpu);
3019                 decache_vcpus_on_cpu(cpu);
3020                 smp_call_function_single(cpu, kvm_arch_ops->hardware_disable,
3021                                          NULL, 0, 1);
3022                 break;
3023         case CPU_ONLINE:
3024         case CPU_ONLINE_FROZEN:
3025                 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
3026                        cpu);
3027                 smp_call_function_single(cpu, kvm_arch_ops->hardware_enable,
3028                                          NULL, 0, 1);
3029                 break;
3030         }
3031         return NOTIFY_OK;
3032 }
3033
3034 static struct notifier_block kvm_cpu_notifier = {
3035         .notifier_call = kvm_cpu_hotplug,
3036         .priority = 20, /* must be > scheduler priority */
3037 };
3038
3039 static u64 stat_get(void *_offset)
3040 {
3041         unsigned offset = (long)_offset;
3042         u64 total = 0;
3043         struct kvm *kvm;
3044         struct kvm_vcpu *vcpu;
3045         int i;
3046
3047         spin_lock(&kvm_lock);
3048         list_for_each_entry(kvm, &vm_list, vm_list)
3049                 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
3050                         vcpu = &kvm->vcpus[i];
3051                         total += *(u32 *)((void *)vcpu + offset);
3052                 }
3053         spin_unlock(&kvm_lock);
3054         return total;
3055 }
3056
3057 static void stat_set(void *offset, u64 val)
3058 {
3059 }
3060
3061 DEFINE_SIMPLE_ATTRIBUTE(stat_fops, stat_get, stat_set, "%llu\n");
3062
3063 static __init void kvm_init_debug(void)
3064 {
3065         struct kvm_stats_debugfs_item *p;
3066
3067         debugfs_dir = debugfs_create_dir("kvm", NULL);
3068         for (p = debugfs_entries; p->name; ++p)
3069                 p->dentry = debugfs_create_file(p->name, 0444, debugfs_dir,
3070                                                 (void *)(long)p->offset,
3071                                                 &stat_fops);
3072 }
3073
3074 static void kvm_exit_debug(void)
3075 {
3076         struct kvm_stats_debugfs_item *p;
3077
3078         for (p = debugfs_entries; p->name; ++p)
3079                 debugfs_remove(p->dentry);
3080         debugfs_remove(debugfs_dir);
3081 }
3082
3083 static int kvm_suspend(struct sys_device *dev, pm_message_t state)
3084 {
3085         decache_vcpus_on_cpu(raw_smp_processor_id());
3086         on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
3087         return 0;
3088 }
3089
3090 static int kvm_resume(struct sys_device *dev)
3091 {
3092         on_each_cpu(kvm_arch_ops->hardware_enable, NULL, 0, 1);
3093         return 0;
3094 }
3095
3096 static struct sysdev_class kvm_sysdev_class = {
3097         set_kset_name("kvm"),
3098         .suspend = kvm_suspend,
3099         .resume = kvm_resume,
3100 };
3101
3102 static struct sys_device kvm_sysdev = {
3103         .id = 0,
3104         .cls = &kvm_sysdev_class,
3105 };
3106
3107 hpa_t bad_page_address;
3108
3109 static int kvmfs_get_sb(struct file_system_type *fs_type, int flags,
3110                         const char *dev_name, void *data, struct vfsmount *mnt)
3111 {
3112         return get_sb_pseudo(fs_type, "kvm:", NULL, KVMFS_SUPER_MAGIC, mnt);
3113 }
3114
3115 static struct file_system_type kvm_fs_type = {
3116         .name           = "kvmfs",
3117         .get_sb         = kvmfs_get_sb,
3118         .kill_sb        = kill_anon_super,
3119 };
3120
3121 int kvm_init_arch(struct kvm_arch_ops *ops, struct module *module)
3122 {
3123         int r;
3124
3125         if (kvm_arch_ops) {
3126                 printk(KERN_ERR "kvm: already loaded the other module\n");
3127                 return -EEXIST;
3128         }
3129
3130         if (!ops->cpu_has_kvm_support()) {
3131                 printk(KERN_ERR "kvm: no hardware support\n");
3132                 return -EOPNOTSUPP;
3133         }
3134         if (ops->disabled_by_bios()) {
3135                 printk(KERN_ERR "kvm: disabled by bios\n");
3136                 return -EOPNOTSUPP;
3137         }
3138
3139         kvm_arch_ops = ops;
3140
3141         r = kvm_arch_ops->hardware_setup();
3142         if (r < 0)
3143                 goto out;
3144
3145         on_each_cpu(kvm_arch_ops->hardware_enable, NULL, 0, 1);
3146         r = register_cpu_notifier(&kvm_cpu_notifier);
3147         if (r)
3148                 goto out_free_1;
3149         register_reboot_notifier(&kvm_reboot_notifier);
3150
3151         r = sysdev_class_register(&kvm_sysdev_class);
3152         if (r)
3153                 goto out_free_2;
3154
3155         r = sysdev_register(&kvm_sysdev);
3156         if (r)
3157                 goto out_free_3;
3158
3159         kvm_chardev_ops.owner = module;
3160
3161         r = misc_register(&kvm_dev);
3162         if (r) {
3163                 printk (KERN_ERR "kvm: misc device register failed\n");
3164                 goto out_free;
3165         }
3166
3167         return r;
3168
3169 out_free:
3170         sysdev_unregister(&kvm_sysdev);
3171 out_free_3:
3172         sysdev_class_unregister(&kvm_sysdev_class);
3173 out_free_2:
3174         unregister_reboot_notifier(&kvm_reboot_notifier);
3175         unregister_cpu_notifier(&kvm_cpu_notifier);
3176 out_free_1:
3177         on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
3178         kvm_arch_ops->hardware_unsetup();
3179 out:
3180         kvm_arch_ops = NULL;
3181         return r;
3182 }
3183
3184 void kvm_exit_arch(void)
3185 {
3186         misc_deregister(&kvm_dev);
3187         sysdev_unregister(&kvm_sysdev);
3188         sysdev_class_unregister(&kvm_sysdev_class);
3189         unregister_reboot_notifier(&kvm_reboot_notifier);
3190         unregister_cpu_notifier(&kvm_cpu_notifier);
3191         on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
3192         kvm_arch_ops->hardware_unsetup();
3193         kvm_arch_ops = NULL;
3194 }
3195
3196 static __init int kvm_init(void)
3197 {
3198         static struct page *bad_page;
3199         int r;
3200
3201         r = kvm_mmu_module_init();
3202         if (r)
3203                 goto out4;
3204
3205         r = register_filesystem(&kvm_fs_type);
3206         if (r)
3207                 goto out3;
3208
3209         kvmfs_mnt = kern_mount(&kvm_fs_type);
3210         r = PTR_ERR(kvmfs_mnt);
3211         if (IS_ERR(kvmfs_mnt))
3212                 goto out2;
3213         kvm_init_debug();
3214
3215         kvm_init_msr_list();
3216
3217         if ((bad_page = alloc_page(GFP_KERNEL)) == NULL) {
3218                 r = -ENOMEM;
3219                 goto out;
3220         }
3221
3222         bad_page_address = page_to_pfn(bad_page) << PAGE_SHIFT;
3223         memset(__va(bad_page_address), 0, PAGE_SIZE);
3224
3225         return 0;
3226
3227 out:
3228         kvm_exit_debug();
3229         mntput(kvmfs_mnt);
3230 out2:
3231         unregister_filesystem(&kvm_fs_type);
3232 out3:
3233         kvm_mmu_module_exit();
3234 out4:
3235         return r;
3236 }
3237
3238 static __exit void kvm_exit(void)
3239 {
3240         kvm_exit_debug();
3241         __free_page(pfn_to_page(bad_page_address >> PAGE_SHIFT));
3242         mntput(kvmfs_mnt);
3243         unregister_filesystem(&kvm_fs_type);
3244         kvm_mmu_module_exit();
3245 }
3246
3247 module_init(kvm_init)
3248 module_exit(kvm_exit)
3249
3250 EXPORT_SYMBOL_GPL(kvm_init_arch);
3251 EXPORT_SYMBOL_GPL(kvm_exit_arch);