drivers/kvm/kvm_main.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  *
   9  * Authors:
  10  *   Avi Kivity   <avi@qumranet.com>
  11  *   Yaniv Kamay  <yaniv@qumranet.com>
  12  *
  13  * This work is licensed under the terms of the GNU GPL, version 2.  See
  14  * the COPYING file in the top-level directory.
  15  *
  16  */
  17
  18 #include "kvm.h"
  19
  20 #include <linux/kvm.h>
  21 #include <linux/module.h>
  22 #include <linux/errno.h>
  23 #include <asm/processor.h>
  24 #include <linux/percpu.h>
  25 #include <linux/gfp.h>
  26 #include <asm/msr.h>
  27 #include <linux/mm.h>
  28 #include <linux/miscdevice.h>
  29 #include <linux/vmalloc.h>
  30 #include <asm/uaccess.h>
  31 #include <linux/reboot.h>
  32 #include <asm/io.h>
  33 #include <linux/debugfs.h>
  34 #include <linux/highmem.h>
  35 #include <linux/file.h>
  36 #include <asm/desc.h>
  37
  38 #include "x86_emulate.h"
  39 #include "segment_descriptor.h"
  40
  41 MODULE_AUTHOR("Qumranet");
  42 MODULE_LICENSE("GPL");
  43
  44 struct kvm_arch_ops *kvm_arch_ops;
  45 struct kvm_stat kvm_stat;
  46 EXPORT_SYMBOL_GPL(kvm_stat);
  47
  48 static struct kvm_stats_debugfs_item {
  49         const char *name;
  50         u32 *data;
  51         struct dentry *dentry;
  52 } debugfs_entries[] = {
  53         { "pf_fixed", &kvm_stat.pf_fixed },
  54         { "pf_guest", &kvm_stat.pf_guest },
  55         { "tlb_flush", &kvm_stat.tlb_flush },
  56         { "invlpg", &kvm_stat.invlpg },
  57         { "exits", &kvm_stat.exits },
  58         { "io_exits", &kvm_stat.io_exits },
  59         { "mmio_exits", &kvm_stat.mmio_exits },
  60         { "signal_exits", &kvm_stat.signal_exits },
  61         { "irq_exits", &kvm_stat.irq_exits },
  62         { 0, 0 }
  63 };
  64
  65 static struct dentry *debugfs_dir;
  66
  67 #define MAX_IO_MSRS 256
  68
  69 #define CR0_RESEVED_BITS 0xffffffff1ffaffc0ULL
  70 #define LMSW_GUEST_MASK 0x0eULL
  71 #define CR4_RESEVED_BITS (~((1ULL << 11) - 1))
  72 #define CR8_RESEVED_BITS (~0x0fULL)
  73 #define EFER_RESERVED_BITS 0xfffffffffffff2fe
  74
  75 #ifdef CONFIG_X86_64
  76 // LDT or TSS descriptor in the GDT. 16 bytes.
  77 struct segment_descriptor_64 {
  78         struct segment_descriptor s;
  79         u32 base_higher;
  80         u32 pad_zero;
  81 };
  82
  83 #endif
  84
  85 unsigned long segment_base(u16 selector)
  86 {
  87         struct descriptor_table gdt;
  88         struct segment_descriptor *d;
  89         unsigned long table_base;
  90         typedef unsigned long ul;
  91         unsigned long v;
  92
  93         if (selector == 0)
  94                 return 0;
  95
  96         asm ("sgdt %0" : "=m"(gdt));
  97         table_base = gdt.base;
  98
  99         if (selector & 4) {           /* from ldt */
 100                 u16 ldt_selector;
 101
 102                 asm ("sldt %0" : "=g"(ldt_selector));
 103                 table_base = segment_base(ldt_selector);
 104         }
 105         d = (struct segment_descriptor *)(table_base + (selector & ~7));
 106         v = d->base_low | ((ul)d->base_mid << 16) | ((ul)d->base_high << 24);
 107 #ifdef CONFIG_X86_64
 108         if (d->system == 0
 109             && (d->type == 2 || d->type == 9 || d->type == 11))
 110                 v |= ((ul)((struct segment_descriptor_64 *)d)->base_higher) << 32;
 111 #endif
 112         return v;
 113 }
 114 EXPORT_SYMBOL_GPL(segment_base);
 115
 116 static inline int valid_vcpu(int n)
 117 {
 118         return likely(n >= 0 && n < KVM_MAX_VCPUS);
 119 }
 120
 121 int kvm_read_guest(struct kvm_vcpu *vcpu,
 122                              gva_t addr,
 123                              unsigned long size,
 124                              void *dest)
 125 {
 126         unsigned char *host_buf = dest;
 127         unsigned long req_size = size;
 128
 129         while (size) {
 130                 hpa_t paddr;
 131                 unsigned now;
 132                 unsigned offset;
 133                 hva_t guest_buf;
 134
 135                 paddr = gva_to_hpa(vcpu, addr);
 136
 137                 if (is_error_hpa(paddr))
 138                         break;
 139
 140                 guest_buf = (hva_t)kmap_atomic(
 141                                         pfn_to_page(paddr >> PAGE_SHIFT),
 142                                         KM_USER0);
 143                 offset = addr & ~PAGE_MASK;
 144                 guest_buf |= offset;
 145                 now = min(size, PAGE_SIZE - offset);
 146                 memcpy(host_buf, (void*)guest_buf, now);
 147                 host_buf += now;
 148                 addr += now;
 149                 size -= now;
 150                 kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
 151         }
 152         return req_size - size;
 153 }
 154 EXPORT_SYMBOL_GPL(kvm_read_guest);
 155
 156 int kvm_write_guest(struct kvm_vcpu *vcpu,
 157                              gva_t addr,
 158                              unsigned long size,
 159                              void *data)
 160 {
 161         unsigned char *host_buf = data;
 162         unsigned long req_size = size;
 163
 164         while (size) {
 165                 hpa_t paddr;
 166                 unsigned now;
 167                 unsigned offset;
 168                 hva_t guest_buf;
 169
 170                 paddr = gva_to_hpa(vcpu, addr);
 171
 172                 if (is_error_hpa(paddr))
 173                         break;
 174
 175                 guest_buf = (hva_t)kmap_atomic(
 176                                 pfn_to_page(paddr >> PAGE_SHIFT), KM_USER0);
 177                 offset = addr & ~PAGE_MASK;
 178                 guest_buf |= offset;
 179                 now = min(size, PAGE_SIZE - offset);
 180                 memcpy((void*)guest_buf, host_buf, now);
 181                 host_buf += now;
 182                 addr += now;
 183                 size -= now;
 184                 kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
 185         }
 186         return req_size - size;
 187 }
 188 EXPORT_SYMBOL_GPL(kvm_write_guest);
 189
 190 static int vcpu_slot(struct kvm_vcpu *vcpu)
 191 {
 192         return vcpu - vcpu->kvm->vcpus;
 193 }
 194
 195 /*
 196  * Switches to specified vcpu, until a matching vcpu_put()
 197  */
 198 static struct kvm_vcpu *vcpu_load(struct kvm *kvm, int vcpu_slot)
 199 {
 200         struct kvm_vcpu *vcpu = &kvm->vcpus[vcpu_slot];
 201
 202         mutex_lock(&vcpu->mutex);
 203         if (unlikely(!vcpu->vmcs)) {
 204                 mutex_unlock(&vcpu->mutex);
 205                 return 0;
 206         }
 207         return kvm_arch_ops->vcpu_load(vcpu);
 208 }
 209
 210 static void vcpu_put(struct kvm_vcpu *vcpu)
 211 {
 212         kvm_arch_ops->vcpu_put(vcpu);
 213         mutex_unlock(&vcpu->mutex);
 214 }
 215
 216 static int kvm_dev_open(struct inode *inode, struct file *filp)
 217 {
 218         struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
 219         int i;
 220
 221         if (!kvm)
 222                 return -ENOMEM;
 223
 224         spin_lock_init(&kvm->lock);
 225         INIT_LIST_HEAD(&kvm->active_mmu_pages);
 226         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 227                 struct kvm_vcpu *vcpu = &kvm->vcpus[i];
 228
 229                 mutex_init(&vcpu->mutex);
 230                 vcpu->mmu.root_hpa = INVALID_PAGE;
 231                 INIT_LIST_HEAD(&vcpu->free_pages);
 232         }
 233         filp->private_data = kvm;
 234         return 0;
 235 }
 236
 237 /*
 238  * Free any memory in @free but not in @dont.
 239  */
 240 static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
 241                                   struct kvm_memory_slot *dont)
 242 {
 243         int i;
 244
 245         if (!dont || free->phys_mem != dont->phys_mem)
 246                 if (free->phys_mem) {
 247                         for (i = 0; i < free->npages; ++i)
 248                                 if (free->phys_mem[i])
 249                                         __free_page(free->phys_mem[i]);
 250                         vfree(free->phys_mem);
 251                 }
 252
 253         if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
 254                 vfree(free->dirty_bitmap);
 255
 256         free->phys_mem = 0;
 257         free->npages = 0;
 258         free->dirty_bitmap = 0;
 259 }
 260
 261 static void kvm_free_physmem(struct kvm *kvm)
 262 {
 263         int i;
 264
 265         for (i = 0; i < kvm->nmemslots; ++i)
 266                 kvm_free_physmem_slot(&kvm->memslots[i], 0);
 267 }
 268
 269 static void kvm_free_vcpu(struct kvm_vcpu *vcpu)
 270 {
 271         kvm_arch_ops->vcpu_free(vcpu);
 272         kvm_mmu_destroy(vcpu);
 273 }
 274
 275 static void kvm_free_vcpus(struct kvm *kvm)
 276 {
 277         unsigned int i;
 278
 279         for (i = 0; i < KVM_MAX_VCPUS; ++i)
 280                 kvm_free_vcpu(&kvm->vcpus[i]);
 281 }
 282
 283 static int kvm_dev_release(struct inode *inode, struct file *filp)
 284 {
 285         struct kvm *kvm = filp->private_data;
 286
 287         kvm_free_vcpus(kvm);
 288         kvm_free_physmem(kvm);
 289         kfree(kvm);
 290         return 0;
 291 }
 292
 293 static void inject_gp(struct kvm_vcpu *vcpu)
 294 {
 295         kvm_arch_ops->inject_gp(vcpu, 0);
 296 }
 297
 298 static int pdptrs_have_reserved_bits_set(struct kvm_vcpu *vcpu,
 299                                          unsigned long cr3)
 300 {
 301         gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
 302         unsigned offset = (cr3 & (PAGE_SIZE-1)) >> 5;
 303         int i;
 304         u64 pdpte;
 305         u64 *pdpt;
 306         struct kvm_memory_slot *memslot;
 307
 308         spin_lock(&vcpu->kvm->lock);
 309         memslot = gfn_to_memslot(vcpu->kvm, pdpt_gfn);
 310         /* FIXME: !memslot - emulate? 0xff? */
 311         pdpt = kmap_atomic(gfn_to_page(memslot, pdpt_gfn), KM_USER0);
 312
 313         for (i = 0; i < 4; ++i) {
 314                 pdpte = pdpt[offset + i];
 315                 if ((pdpte & 1) && (pdpte & 0xfffffff0000001e6ull))
 316                         break;
 317         }
 318
 319         kunmap_atomic(pdpt, KM_USER0);
 320         spin_unlock(&vcpu->kvm->lock);
 321
 322         return i != 4;
 323 }
 324
 325 void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 326 {
 327         if (cr0 & CR0_RESEVED_BITS) {
 328                 printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
 329                        cr0, vcpu->cr0);
 330                 inject_gp(vcpu);
 331                 return;
 332         }
 333
 334         if ((cr0 & CR0_NW_MASK) && !(cr0 & CR0_CD_MASK)) {
 335                 printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n");
 336                 inject_gp(vcpu);
 337                 return;
 338         }
 339
 340         if ((cr0 & CR0_PG_MASK) && !(cr0 & CR0_PE_MASK)) {
 341                 printk(KERN_DEBUG "set_cr0: #GP, set PG flag "
 342                        "and a clear PE flag\n");
 343                 inject_gp(vcpu);
 344                 return;
 345         }
 346
 347         if (!is_paging(vcpu) && (cr0 & CR0_PG_MASK)) {
 348 #ifdef CONFIG_X86_64
 349                 if ((vcpu->shadow_efer & EFER_LME)) {
 350                         int cs_db, cs_l;
 351
 352                         if (!is_pae(vcpu)) {
 353                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
 354                                        "in long mode while PAE is disabled\n");
 355                                 inject_gp(vcpu);
 356                                 return;
 357                         }
 358                         kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
 359                         if (cs_l) {
 360                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
 361                                        "in long mode while CS.L == 1\n");
 362                                 inject_gp(vcpu);
 363                                 return;
 364
 365                         }
 366                 } else
 367 #endif
 368                 if (is_pae(vcpu) &&
 369                             pdptrs_have_reserved_bits_set(vcpu, vcpu->cr3)) {
 370                         printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
 371                                "reserved bits\n");
 372                         inject_gp(vcpu);
 373                         return;
 374                 }
 375
 376         }
 377
 378         kvm_arch_ops->set_cr0(vcpu, cr0);
 379         vcpu->cr0 = cr0;
 380
 381         spin_lock(&vcpu->kvm->lock);
 382         kvm_mmu_reset_context(vcpu);
 383         spin_unlock(&vcpu->kvm->lock);
 384         return;
 385 }
 386 EXPORT_SYMBOL_GPL(set_cr0);
 387
 388 void lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
 389 {
 390         set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f));
 391 }
 392 EXPORT_SYMBOL_GPL(lmsw);
 393
 394 void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 395 {
 396         if (cr4 & CR4_RESEVED_BITS) {
 397                 printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n");
 398                 inject_gp(vcpu);
 399                 return;
 400         }
 401
 402         if (is_long_mode(vcpu)) {
 403                 if (!(cr4 & CR4_PAE_MASK)) {
 404                         printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while "
 405                                "in long mode\n");
 406                         inject_gp(vcpu);
 407                         return;
 408                 }
 409         } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & CR4_PAE_MASK)
 410                    && pdptrs_have_reserved_bits_set(vcpu, vcpu->cr3)) {
 411                 printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
 412                 inject_gp(vcpu);
 413         }
 414
 415         if (cr4 & CR4_VMXE_MASK) {
 416                 printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n");
 417                 inject_gp(vcpu);
 418                 return;
 419         }
 420         kvm_arch_ops->set_cr4(vcpu, cr4);
 421         spin_lock(&vcpu->kvm->lock);
 422         kvm_mmu_reset_context(vcpu);
 423         spin_unlock(&vcpu->kvm->lock);
 424 }
 425 EXPORT_SYMBOL_GPL(set_cr4);
 426
 427 void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
 428 {
 429         if (is_long_mode(vcpu)) {
 430                 if ( cr3 & CR3_L_MODE_RESEVED_BITS) {
 431                         printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
 432                         inject_gp(vcpu);
 433                         return;
 434                 }
 435         } else {
 436                 if (cr3 & CR3_RESEVED_BITS) {
 437                         printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
 438                         inject_gp(vcpu);
 439                         return;
 440                 }
 441                 if (is_paging(vcpu) && is_pae(vcpu) &&
 442                     pdptrs_have_reserved_bits_set(vcpu, cr3)) {
 443                         printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
 444                                "reserved bits\n");
 445                         inject_gp(vcpu);
 446                         return;
 447                 }
 448         }
 449
 450         vcpu->cr3 = cr3;
 451         spin_lock(&vcpu->kvm->lock);
 452         vcpu->mmu.new_cr3(vcpu);
 453         spin_unlock(&vcpu->kvm->lock);
 454 }
 455 EXPORT_SYMBOL_GPL(set_cr3);
 456
 457 void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
 458 {
 459         if ( cr8 & CR8_RESEVED_BITS) {
 460                 printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8);
 461                 inject_gp(vcpu);
 462                 return;
 463         }
 464         vcpu->cr8 = cr8;
 465 }
 466 EXPORT_SYMBOL_GPL(set_cr8);
 467
 468 void fx_init(struct kvm_vcpu *vcpu)
 469 {
 470         struct __attribute__ ((__packed__)) fx_image_s {
 471                 u16 control; //fcw
 472                 u16 status; //fsw
 473                 u16 tag; // ftw
 474                 u16 opcode; //fop
 475                 u64 ip; // fpu ip
 476                 u64 operand;// fpu dp
 477                 u32 mxcsr;
 478                 u32 mxcsr_mask;
 479
 480         } *fx_image;
 481
 482         fx_save(vcpu->host_fx_image);
 483         fpu_init();
 484         fx_save(vcpu->guest_fx_image);
 485         fx_restore(vcpu->host_fx_image);
 486
 487         fx_image = (struct fx_image_s *)vcpu->guest_fx_image;
 488         fx_image->mxcsr = 0x1f80;
 489         memset(vcpu->guest_fx_image + sizeof(struct fx_image_s),
 490                0, FX_IMAGE_SIZE - sizeof(struct fx_image_s));
 491 }
 492 EXPORT_SYMBOL_GPL(fx_init);
 493
 494 /*
 495  * Creates some virtual cpus.  Good luck creating more than one.
 496  */
 497 static int kvm_dev_ioctl_create_vcpu(struct kvm *kvm, int n)
 498 {
 499         int r;
 500         struct kvm_vcpu *vcpu;
 501
 502         r = -EINVAL;
 503         if (!valid_vcpu(n))
 504                 goto out;
 505
 506         vcpu = &kvm->vcpus[n];
 507
 508         mutex_lock(&vcpu->mutex);
 509
 510         if (vcpu->vmcs) {
 511                 mutex_unlock(&vcpu->mutex);
 512                 return -EEXIST;
 513         }
 514
 515         vcpu->host_fx_image = (char*)ALIGN((hva_t)vcpu->fx_buf,
 516                                            FX_IMAGE_ALIGN);
 517         vcpu->guest_fx_image = vcpu->host_fx_image + FX_IMAGE_SIZE;
 518
 519         vcpu->cpu = -1;  /* First load will set up TR */
 520         vcpu->kvm = kvm;
 521         r = kvm_arch_ops->vcpu_create(vcpu);
 522         if (r < 0)
 523                 goto out_free_vcpus;
 524
 525         r = kvm_mmu_create(vcpu);
 526         if (r < 0)
 527                 goto out_free_vcpus;
 528
 529         kvm_arch_ops->vcpu_load(vcpu);
 530         r = kvm_mmu_setup(vcpu);
 531         if (r >= 0)
 532                 r = kvm_arch_ops->vcpu_setup(vcpu);
 533         vcpu_put(vcpu);
 534
 535         if (r < 0)
 536                 goto out_free_vcpus;
 537
 538         return 0;
 539
 540 out_free_vcpus:
 541         kvm_free_vcpu(vcpu);
 542         mutex_unlock(&vcpu->mutex);
 543 out:
 544         return r;
 545 }
 546
 547 /*
 548  * Allocate some memory and give it an address in the guest physical address
 549  * space.
 550  *
 551  * Discontiguous memory is allowed, mostly for framebuffers.
 552  */
 553 static int kvm_dev_ioctl_set_memory_region(struct kvm *kvm,
 554                                            struct kvm_memory_region *mem)
 555 {
 556         int r;
 557         gfn_t base_gfn;
 558         unsigned long npages;
 559         unsigned long i;
 560         struct kvm_memory_slot *memslot;
 561         struct kvm_memory_slot old, new;
 562         int memory_config_version;
 563
 564         r = -EINVAL;
 565         /* General sanity checks */
 566         if (mem->memory_size & (PAGE_SIZE - 1))
 567                 goto out;
 568         if (mem->guest_phys_addr & (PAGE_SIZE - 1))
 569                 goto out;
 570         if (mem->slot >= KVM_MEMORY_SLOTS)
 571                 goto out;
 572         if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
 573                 goto out;
 574
 575         memslot = &kvm->memslots[mem->slot];
 576         base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
 577         npages = mem->memory_size >> PAGE_SHIFT;
 578
 579         if (!npages)
 580                 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
 581
 582 raced:
 583         spin_lock(&kvm->lock);
 584
 585         memory_config_version = kvm->memory_config_version;
 586         new = old = *memslot;
 587
 588         new.base_gfn = base_gfn;
 589         new.npages = npages;
 590         new.flags = mem->flags;
 591
 592         /* Disallow changing a memory slot's size. */
 593         r = -EINVAL;
 594         if (npages && old.npages && npages != old.npages)
 595                 goto out_unlock;
 596
 597         /* Check for overlaps */
 598         r = -EEXIST;
 599         for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
 600                 struct kvm_memory_slot *s = &kvm->memslots[i];
 601
 602                 if (s == memslot)
 603                         continue;
 604                 if (!((base_gfn + npages <= s->base_gfn) ||
 605                       (base_gfn >= s->base_gfn + s->npages)))
 606                         goto out_unlock;
 607         }
 608         /*
 609          * Do memory allocations outside lock.  memory_config_version will
 610          * detect any races.
 611          */
 612         spin_unlock(&kvm->lock);
 613
 614         /* Deallocate if slot is being removed */
 615         if (!npages)
 616                 new.phys_mem = 0;
 617
 618         /* Free page dirty bitmap if unneeded */
 619         if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
 620                 new.dirty_bitmap = 0;
 621
 622         r = -ENOMEM;
 623
 624         /* Allocate if a slot is being created */
 625         if (npages && !new.phys_mem) {
 626                 new.phys_mem = vmalloc(npages * sizeof(struct page *));
 627
 628                 if (!new.phys_mem)
 629                         goto out_free;
 630
 631                 memset(new.phys_mem, 0, npages * sizeof(struct page *));
 632                 for (i = 0; i < npages; ++i) {
 633                         new.phys_mem[i] = alloc_page(GFP_HIGHUSER
 634                                                      | __GFP_ZERO);
 635                         if (!new.phys_mem[i])
 636                                 goto out_free;
 637                 }
 638         }
 639
 640         /* Allocate page dirty bitmap if needed */
 641         if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
 642                 unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;
 643
 644                 new.dirty_bitmap = vmalloc(dirty_bytes);
 645                 if (!new.dirty_bitmap)
 646                         goto out_free;
 647                 memset(new.dirty_bitmap, 0, dirty_bytes);
 648         }
 649
 650         spin_lock(&kvm->lock);
 651
 652         if (memory_config_version != kvm->memory_config_version) {
 653                 spin_unlock(&kvm->lock);
 654                 kvm_free_physmem_slot(&new, &old);
 655                 goto raced;
 656         }
 657
 658         r = -EAGAIN;
 659         if (kvm->busy)
 660                 goto out_unlock;
 661
 662         if (mem->slot >= kvm->nmemslots)
 663                 kvm->nmemslots = mem->slot + 1;
 664
 665         *memslot = new;
 666         ++kvm->memory_config_version;
 667
 668         spin_unlock(&kvm->lock);
 669
 670         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 671                 struct kvm_vcpu *vcpu;
 672
 673                 vcpu = vcpu_load(kvm, i);
 674                 if (!vcpu)
 675                         continue;
 676                 kvm_mmu_reset_context(vcpu);
 677                 vcpu_put(vcpu);
 678         }
 679
 680         kvm_free_physmem_slot(&old, &new);
 681         return 0;
 682
 683 out_unlock:
 684         spin_unlock(&kvm->lock);
 685 out_free:
 686         kvm_free_physmem_slot(&new, &old);
 687 out:
 688         return r;
 689 }
 690
 691 /*
 692  * Get (and clear) the dirty memory log for a memory slot.
 693  */
 694 static int kvm_dev_ioctl_get_dirty_log(struct kvm *kvm,
 695                                        struct kvm_dirty_log *log)
 696 {
 697         struct kvm_memory_slot *memslot;
 698         int r, i;
 699         int n;
 700         unsigned long any = 0;
 701
 702         spin_lock(&kvm->lock);
 703
 704         /*
 705          * Prevent changes to guest memory configuration even while the lock
 706          * is not taken.
 707          */
 708         ++kvm->busy;
 709         spin_unlock(&kvm->lock);
 710         r = -EINVAL;
 711         if (log->slot >= KVM_MEMORY_SLOTS)
 712                 goto out;
 713
 714         memslot = &kvm->memslots[log->slot];
 715         r = -ENOENT;
 716         if (!memslot->dirty_bitmap)
 717                 goto out;
 718
 719         n = ALIGN(memslot->npages, 8) / 8;
 720
 721         for (i = 0; !any && i < n; ++i)
 722                 any = memslot->dirty_bitmap[i];
 723
 724         r = -EFAULT;
 725         if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
 726                 goto out;
 727
 728
 729         if (any) {
 730                 spin_lock(&kvm->lock);
 731                 kvm_mmu_slot_remove_write_access(kvm, log->slot);
 732                 spin_unlock(&kvm->lock);
 733                 memset(memslot->dirty_bitmap, 0, n);
 734                 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 735                         struct kvm_vcpu *vcpu = vcpu_load(kvm, i);
 736
 737                         if (!vcpu)
 738                                 continue;
 739                         kvm_arch_ops->tlb_flush(vcpu);
 740                         vcpu_put(vcpu);
 741                 }
 742         }
 743
 744         r = 0;
 745
 746 out:
 747         spin_lock(&kvm->lock);
 748         --kvm->busy;
 749         spin_unlock(&kvm->lock);
 750         return r;
 751 }
 752
 753 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
 754 {
 755         int i;
 756
 757         for (i = 0; i < kvm->nmemslots; ++i) {
 758                 struct kvm_memory_slot *memslot = &kvm->memslots[i];
 759
 760                 if (gfn >= memslot->base_gfn
 761                     && gfn < memslot->base_gfn + memslot->npages)
 762                         return memslot;
 763         }
 764         return 0;
 765 }
 766 EXPORT_SYMBOL_GPL(gfn_to_memslot);
 767
 768 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
 769 {
 770         int i;
 771         struct kvm_memory_slot *memslot = 0;
 772         unsigned long rel_gfn;
 773
 774         for (i = 0; i < kvm->nmemslots; ++i) {
 775                 memslot = &kvm->memslots[i];
 776
 777                 if (gfn >= memslot->base_gfn
 778                     && gfn < memslot->base_gfn + memslot->npages) {
 779
 780                         if (!memslot || !memslot->dirty_bitmap)
 781                                 return;
 782
 783                         rel_gfn = gfn - memslot->base_gfn;
 784
 785                         /* avoid RMW */
 786                         if (!test_bit(rel_gfn, memslot->dirty_bitmap))
 787                                 set_bit(rel_gfn, memslot->dirty_bitmap);
 788                         return;
 789                 }
 790         }
 791 }
 792
 793 static int emulator_read_std(unsigned long addr,
 794                              unsigned long *val,
 795                              unsigned int bytes,
 796                              struct x86_emulate_ctxt *ctxt)
 797 {
 798         struct kvm_vcpu *vcpu = ctxt->vcpu;
 799         void *data = val;
 800
 801         while (bytes) {
 802                 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
 803                 unsigned offset = addr & (PAGE_SIZE-1);
 804                 unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset);
 805                 unsigned long pfn;
 806                 struct kvm_memory_slot *memslot;
 807                 void *page;
 808
 809                 if (gpa == UNMAPPED_GVA)
 810                         return X86EMUL_PROPAGATE_FAULT;
 811                 pfn = gpa >> PAGE_SHIFT;
 812                 memslot = gfn_to_memslot(vcpu->kvm, pfn);
 813                 if (!memslot)
 814                         return X86EMUL_UNHANDLEABLE;
 815                 page = kmap_atomic(gfn_to_page(memslot, pfn), KM_USER0);
 816
 817                 memcpy(data, page + offset, tocopy);
 818
 819                 kunmap_atomic(page, KM_USER0);
 820
 821                 bytes -= tocopy;
 822                 data += tocopy;
 823                 addr += tocopy;
 824         }
 825
 826         return X86EMUL_CONTINUE;
 827 }
 828
 829 static int emulator_write_std(unsigned long addr,
 830                               unsigned long val,
 831                               unsigned int bytes,
 832                               struct x86_emulate_ctxt *ctxt)
 833 {
 834         printk(KERN_ERR "emulator_write_std: addr %lx n %d\n",
 835                addr, bytes);
 836         return X86EMUL_UNHANDLEABLE;
 837 }
 838
 839 static int emulator_read_emulated(unsigned long addr,
 840                                   unsigned long *val,
 841                                   unsigned int bytes,
 842                                   struct x86_emulate_ctxt *ctxt)
 843 {
 844         struct kvm_vcpu *vcpu = ctxt->vcpu;
 845
 846         if (vcpu->mmio_read_completed) {
 847                 memcpy(val, vcpu->mmio_data, bytes);
 848                 vcpu->mmio_read_completed = 0;
 849                 return X86EMUL_CONTINUE;
 850         } else if (emulator_read_std(addr, val, bytes, ctxt)
 851                    == X86EMUL_CONTINUE)
 852                 return X86EMUL_CONTINUE;
 853         else {
 854                 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
 855                 if (gpa == UNMAPPED_GVA)
 856                         return vcpu_printf(vcpu, "not present\n"), X86EMUL_PROPAGATE_FAULT;
 857                 vcpu->mmio_needed = 1;
 858                 vcpu->mmio_phys_addr = gpa;
 859                 vcpu->mmio_size = bytes;
 860                 vcpu->mmio_is_write = 0;
 861
 862                 return X86EMUL_UNHANDLEABLE;
 863         }
 864 }
 865
 866 static int emulator_write_emulated(unsigned long addr,
 867                                    unsigned long val,
 868                                    unsigned int bytes,
 869                                    struct x86_emulate_ctxt *ctxt)
 870 {
 871         struct kvm_vcpu *vcpu = ctxt->vcpu;
 872         gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
 873
 874         if (gpa == UNMAPPED_GVA)
 875                 return X86EMUL_PROPAGATE_FAULT;
 876
 877         vcpu->mmio_needed = 1;
 878         vcpu->mmio_phys_addr = gpa;
 879         vcpu->mmio_size = bytes;
 880         vcpu->mmio_is_write = 1;
 881         memcpy(vcpu->mmio_data, &val, bytes);
 882
 883         return X86EMUL_CONTINUE;
 884 }
 885
 886 static int emulator_cmpxchg_emulated(unsigned long addr,
 887                                      unsigned long old,
 888                                      unsigned long new,
 889                                      unsigned int bytes,
 890                                      struct x86_emulate_ctxt *ctxt)
 891 {
 892         static int reported;
 893
 894         if (!reported) {
 895                 reported = 1;
 896                 printk(KERN_WARNING "kvm: emulating exchange as write\n");
 897         }
 898         return emulator_write_emulated(addr, new, bytes, ctxt);
 899 }
 900
 901 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
 902 {
 903         return kvm_arch_ops->get_segment_base(vcpu, seg);
 904 }
 905
 906 int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
 907 {
 908         spin_lock(&vcpu->kvm->lock);
 909         vcpu->mmu.inval_page(vcpu, address);
 910         spin_unlock(&vcpu->kvm->lock);
 911         kvm_arch_ops->invlpg(vcpu, address);
 912         return X86EMUL_CONTINUE;
 913 }
 914
 915 int emulate_clts(struct kvm_vcpu *vcpu)
 916 {
 917         unsigned long cr0 = vcpu->cr0;
 918
 919         cr0 &= ~CR0_TS_MASK;
 920         kvm_arch_ops->set_cr0(vcpu, cr0);
 921         return X86EMUL_CONTINUE;
 922 }
 923
 924 int emulator_get_dr(struct x86_emulate_ctxt* ctxt, int dr, unsigned long *dest)
 925 {
 926         struct kvm_vcpu *vcpu = ctxt->vcpu;
 927
 928         switch (dr) {
 929         case 0 ... 3:
 930                 *dest = kvm_arch_ops->get_dr(vcpu, dr);
 931                 return X86EMUL_CONTINUE;
 932         default:
 933                 printk(KERN_DEBUG "%s: unexpected dr %u\n",
 934                        __FUNCTION__, dr);
 935                 return X86EMUL_UNHANDLEABLE;
 936         }
 937 }
 938
 939 int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
 940 {
 941         unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
 942         int exception;
 943
 944         kvm_arch_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception);
 945         if (exception) {
 946                 /* FIXME: better handling */
 947                 return X86EMUL_UNHANDLEABLE;
 948         }
 949         return X86EMUL_CONTINUE;
 950 }
 951
 952 static void report_emulation_failure(struct x86_emulate_ctxt *ctxt)
 953 {
 954         static int reported;
 955         u8 opcodes[4];
 956         unsigned long rip = ctxt->vcpu->rip;
 957         unsigned long rip_linear;
 958
 959         rip_linear = rip + get_segment_base(ctxt->vcpu, VCPU_SREG_CS);
 960
 961         if (reported)
 962                 return;
 963
 964         emulator_read_std(rip_linear, (void *)opcodes, 4, ctxt);
 965
 966         printk(KERN_ERR "emulation failed but !mmio_needed?"
 967                " rip %lx %02x %02x %02x %02x\n",
 968                rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]);
 969         reported = 1;
 970 }
 971
 972 struct x86_emulate_ops emulate_ops = {
 973         .read_std            = emulator_read_std,
 974         .write_std           = emulator_write_std,
 975         .read_emulated       = emulator_read_emulated,
 976         .write_emulated      = emulator_write_emulated,
 977         .cmpxchg_emulated    = emulator_cmpxchg_emulated,
 978 };
 979
 980 int emulate_instruction(struct kvm_vcpu *vcpu,
 981                         struct kvm_run *run,
 982                         unsigned long cr2,
 983                         u16 error_code)
 984 {
 985         struct x86_emulate_ctxt emulate_ctxt;
 986         int r;
 987         int cs_db, cs_l;
 988
 989         kvm_arch_ops->cache_regs(vcpu);
 990
 991         kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
 992
 993         emulate_ctxt.vcpu = vcpu;
 994         emulate_ctxt.eflags = kvm_arch_ops->get_rflags(vcpu);
 995         emulate_ctxt.cr2 = cr2;
 996         emulate_ctxt.mode = (emulate_ctxt.eflags & X86_EFLAGS_VM)
 997                 ? X86EMUL_MODE_REAL : cs_l
 998                 ? X86EMUL_MODE_PROT64 : cs_db
 999                 ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
1000
1001         if (emulate_ctxt.mode == X86EMUL_MODE_PROT64) {
1002                 emulate_ctxt.cs_base = 0;
1003                 emulate_ctxt.ds_base = 0;
1004                 emulate_ctxt.es_base = 0;
1005                 emulate_ctxt.ss_base = 0;
1006         } else {
1007                 emulate_ctxt.cs_base = get_segment_base(vcpu, VCPU_SREG_CS);
1008                 emulate_ctxt.ds_base = get_segment_base(vcpu, VCPU_SREG_DS);
1009                 emulate_ctxt.es_base = get_segment_base(vcpu, VCPU_SREG_ES);
1010                 emulate_ctxt.ss_base = get_segment_base(vcpu, VCPU_SREG_SS);
1011         }
1012
1013         emulate_ctxt.gs_base = get_segment_base(vcpu, VCPU_SREG_GS);
1014         emulate_ctxt.fs_base = get_segment_base(vcpu, VCPU_SREG_FS);
1015
1016         vcpu->mmio_is_write = 0;
1017         r = x86_emulate_memop(&emulate_ctxt, &emulate_ops);
1018
1019         if ((r || vcpu->mmio_is_write) && run) {
1020                 run->mmio.phys_addr = vcpu->mmio_phys_addr;
1021                 memcpy(run->mmio.data, vcpu->mmio_data, 8);
1022                 run->mmio.len = vcpu->mmio_size;
1023                 run->mmio.is_write = vcpu->mmio_is_write;
1024         }
1025
1026         if (r) {
1027                 if (!vcpu->mmio_needed) {
1028                         report_emulation_failure(&emulate_ctxt);
1029                         return EMULATE_FAIL;
1030                 }
1031                 return EMULATE_DO_MMIO;
1032         }
1033
1034         kvm_arch_ops->decache_regs(vcpu);
1035         kvm_arch_ops->set_rflags(vcpu, emulate_ctxt.eflags);
1036
1037         if (vcpu->mmio_is_write)
1038                 return EMULATE_DO_MMIO;
1039
1040         return EMULATE_DONE;
1041 }
1042 EXPORT_SYMBOL_GPL(emulate_instruction);
1043
1044 static u64 mk_cr_64(u64 curr_cr, u32 new_val)
1045 {
1046         return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
1047 }
1048
1049 void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1050 {
1051         struct descriptor_table dt = { limit, base };
1052
1053         kvm_arch_ops->set_gdt(vcpu, &dt);
1054 }
1055
1056 void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1057 {
1058         struct descriptor_table dt = { limit, base };
1059
1060         kvm_arch_ops->set_idt(vcpu, &dt);
1061 }
1062
1063 void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
1064                    unsigned long *rflags)
1065 {
1066         lmsw(vcpu, msw);
1067         *rflags = kvm_arch_ops->get_rflags(vcpu);
1068 }
1069
1070 unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
1071 {
1072         switch (cr) {
1073         case 0:
1074                 return vcpu->cr0;
1075         case 2:
1076                 return vcpu->cr2;
1077         case 3:
1078                 return vcpu->cr3;
1079         case 4:
1080                 return vcpu->cr4;
1081         default:
1082                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1083                 return 0;
1084         }
1085 }
1086
1087 void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
1088                      unsigned long *rflags)
1089 {
1090         switch (cr) {
1091         case 0:
1092                 set_cr0(vcpu, mk_cr_64(vcpu->cr0, val));
1093                 *rflags = kvm_arch_ops->get_rflags(vcpu);
1094                 break;
1095         case 2:
1096                 vcpu->cr2 = val;
1097                 break;
1098         case 3:
1099                 set_cr3(vcpu, val);
1100                 break;
1101         case 4:
1102                 set_cr4(vcpu, mk_cr_64(vcpu->cr4, val));
1103                 break;
1104         default:
1105                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1106         }
1107 }
1108
1109 int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
1110 {
1111         u64 data;
1112
1113         switch (msr) {
1114         case 0xc0010010: /* SYSCFG */
1115         case 0xc0010015: /* HWCR */
1116         case MSR_IA32_PLATFORM_ID:
1117         case MSR_IA32_P5_MC_ADDR:
1118         case MSR_IA32_P5_MC_TYPE:
1119         case MSR_IA32_MC0_CTL:
1120         case MSR_IA32_MCG_STATUS:
1121         case MSR_IA32_MCG_CAP:
1122         case MSR_IA32_MC0_MISC:
1123         case MSR_IA32_MC0_MISC+4:
1124         case MSR_IA32_MC0_MISC+8:
1125         case MSR_IA32_MC0_MISC+12:
1126         case MSR_IA32_MC0_MISC+16:
1127         case MSR_IA32_UCODE_REV:
1128         case MSR_IA32_PERF_STATUS:
1129                 /* MTRR registers */
1130         case 0xfe:
1131         case 0x200 ... 0x2ff:
1132                 data = 0;
1133                 break;
1134         case 0xcd: /* fsb frequency */
1135                 data = 3;
1136                 break;
1137         case MSR_IA32_APICBASE:
1138                 data = vcpu->apic_base;
1139                 break;
1140 #ifdef CONFIG_X86_64
1141         case MSR_EFER:
1142                 data = vcpu->shadow_efer;
1143                 break;
1144 #endif
1145         default:
1146                 printk(KERN_ERR "kvm: unhandled rdmsr: 0x%x\n", msr);
1147                 return 1;
1148         }
1149         *pdata = data;
1150         return 0;
1151 }
1152 EXPORT_SYMBOL_GPL(kvm_get_msr_common);
1153
1154 /*
1155  * Reads an msr value (of 'msr_index') into 'pdata'.
1156  * Returns 0 on success, non-0 otherwise.
1157  * Assumes vcpu_load() was already called.
1158  */
1159 static int get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
1160 {
1161         return kvm_arch_ops->get_msr(vcpu, msr_index, pdata);
1162 }
1163
1164 #ifdef CONFIG_X86_64
1165
1166 static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
1167 {
1168         if (efer & EFER_RESERVED_BITS) {
1169                 printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n",
1170                        efer);
1171                 inject_gp(vcpu);
1172                 return;
1173         }
1174
1175         if (is_paging(vcpu)
1176             && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) {
1177                 printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n");
1178                 inject_gp(vcpu);
1179                 return;
1180         }
1181
1182         kvm_arch_ops->set_efer(vcpu, efer);
1183
1184         efer &= ~EFER_LMA;
1185         efer |= vcpu->shadow_efer & EFER_LMA;
1186
1187         vcpu->shadow_efer = efer;
1188 }
1189
1190 #endif
1191
1192 int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
1193 {
1194         switch (msr) {
1195 #ifdef CONFIG_X86_64
1196         case MSR_EFER:
1197                 set_efer(vcpu, data);
1198                 break;
1199 #endif
1200         case MSR_IA32_MC0_STATUS:
1201                 printk(KERN_WARNING "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n",
1202                        __FUNCTION__, data);
1203                 break;
1204         case MSR_IA32_UCODE_REV:
1205         case MSR_IA32_UCODE_WRITE:
1206         case 0x200 ... 0x2ff: /* MTRRs */
1207                 break;
1208         case MSR_IA32_APICBASE:
1209                 vcpu->apic_base = data;
1210                 break;
1211         default:
1212                 printk(KERN_ERR "kvm: unhandled wrmsr: 0x%x\n", msr);
1213                 return 1;
1214         }
1215         return 0;
1216 }
1217 EXPORT_SYMBOL_GPL(kvm_set_msr_common);
1218
1219 /*
1220  * Writes msr value into into the appropriate "register".
1221  * Returns 0 on success, non-0 otherwise.
1222  * Assumes vcpu_load() was already called.
1223  */
1224 static int set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
1225 {
1226         return kvm_arch_ops->set_msr(vcpu, msr_index, data);
1227 }
1228
1229 void kvm_resched(struct kvm_vcpu *vcpu)
1230 {
1231         vcpu_put(vcpu);
1232         cond_resched();
1233         /* Cannot fail -  no vcpu unplug yet. */
1234         vcpu_load(vcpu->kvm, vcpu_slot(vcpu));
1235 }
1236 EXPORT_SYMBOL_GPL(kvm_resched);
1237
1238 void load_msrs(struct vmx_msr_entry *e, int n)
1239 {
1240         int i;
1241
1242         for (i = 0; i < n; ++i)
1243                 wrmsrl(e[i].index, e[i].data);
1244 }
1245 EXPORT_SYMBOL_GPL(load_msrs);
1246
1247 void save_msrs(struct vmx_msr_entry *e, int n)
1248 {
1249         int i;
1250
1251         for (i = 0; i < n; ++i)
1252                 rdmsrl(e[i].index, e[i].data);
1253 }
1254 EXPORT_SYMBOL_GPL(save_msrs);
1255
1256 static int kvm_dev_ioctl_run(struct kvm *kvm, struct kvm_run *kvm_run)
1257 {
1258         struct kvm_vcpu *vcpu;
1259         int r;
1260
1261         if (!valid_vcpu(kvm_run->vcpu))
1262                 return -EINVAL;
1263
1264         vcpu = vcpu_load(kvm, kvm_run->vcpu);
1265         if (!vcpu)
1266                 return -ENOENT;
1267
1268         if (kvm_run->emulated) {
1269                 kvm_arch_ops->skip_emulated_instruction(vcpu);
1270                 kvm_run->emulated = 0;
1271         }
1272
1273         if (kvm_run->mmio_completed) {
1274                 memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
1275                 vcpu->mmio_read_completed = 1;
1276         }
1277
1278         vcpu->mmio_needed = 0;
1279
1280         r = kvm_arch_ops->run(vcpu, kvm_run);
1281
1282         vcpu_put(vcpu);
1283         return r;
1284 }
1285
1286 static int kvm_dev_ioctl_get_regs(struct kvm *kvm, struct kvm_regs *regs)
1287 {
1288         struct kvm_vcpu *vcpu;
1289
1290         if (!valid_vcpu(regs->vcpu))
1291                 return -EINVAL;
1292
1293         vcpu = vcpu_load(kvm, regs->vcpu);
1294         if (!vcpu)
1295                 return -ENOENT;
1296
1297         kvm_arch_ops->cache_regs(vcpu);
1298
1299         regs->rax = vcpu->regs[VCPU_REGS_RAX];
1300         regs->rbx = vcpu->regs[VCPU_REGS_RBX];
1301         regs->rcx = vcpu->regs[VCPU_REGS_RCX];
1302         regs->rdx = vcpu->regs[VCPU_REGS_RDX];
1303         regs->rsi = vcpu->regs[VCPU_REGS_RSI];
1304         regs->rdi = vcpu->regs[VCPU_REGS_RDI];
1305         regs->rsp = vcpu->regs[VCPU_REGS_RSP];
1306         regs->rbp = vcpu->regs[VCPU_REGS_RBP];
1307 #ifdef CONFIG_X86_64
1308         regs->r8 = vcpu->regs[VCPU_REGS_R8];
1309         regs->r9 = vcpu->regs[VCPU_REGS_R9];
1310         regs->r10 = vcpu->regs[VCPU_REGS_R10];
1311         regs->r11 = vcpu->regs[VCPU_REGS_R11];
1312         regs->r12 = vcpu->regs[VCPU_REGS_R12];
1313         regs->r13 = vcpu->regs[VCPU_REGS_R13];
1314         regs->r14 = vcpu->regs[VCPU_REGS_R14];
1315         regs->r15 = vcpu->regs[VCPU_REGS_R15];
1316 #endif
1317
1318         regs->rip = vcpu->rip;
1319         regs->rflags = kvm_arch_ops->get_rflags(vcpu);
1320
1321         /*
1322          * Don't leak debug flags in case they were set for guest debugging
1323          */
1324         if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep)
1325                 regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
1326
1327         vcpu_put(vcpu);
1328
1329         return 0;
1330 }
1331
1332 static int kvm_dev_ioctl_set_regs(struct kvm *kvm, struct kvm_regs *regs)
1333 {
1334         struct kvm_vcpu *vcpu;
1335
1336         if (!valid_vcpu(regs->vcpu))
1337                 return -EINVAL;
1338
1339         vcpu = vcpu_load(kvm, regs->vcpu);
1340         if (!vcpu)
1341                 return -ENOENT;
1342
1343         vcpu->regs[VCPU_REGS_RAX] = regs->rax;
1344         vcpu->regs[VCPU_REGS_RBX] = regs->rbx;
1345         vcpu->regs[VCPU_REGS_RCX] = regs->rcx;
1346         vcpu->regs[VCPU_REGS_RDX] = regs->rdx;
1347         vcpu->regs[VCPU_REGS_RSI] = regs->rsi;
1348         vcpu->regs[VCPU_REGS_RDI] = regs->rdi;
1349         vcpu->regs[VCPU_REGS_RSP] = regs->rsp;
1350         vcpu->regs[VCPU_REGS_RBP] = regs->rbp;
1351 #ifdef CONFIG_X86_64
1352         vcpu->regs[VCPU_REGS_R8] = regs->r8;
1353         vcpu->regs[VCPU_REGS_R9] = regs->r9;
1354         vcpu->regs[VCPU_REGS_R10] = regs->r10;
1355         vcpu->regs[VCPU_REGS_R11] = regs->r11;
1356         vcpu->regs[VCPU_REGS_R12] = regs->r12;
1357         vcpu->regs[VCPU_REGS_R13] = regs->r13;
1358         vcpu->regs[VCPU_REGS_R14] = regs->r14;
1359         vcpu->regs[VCPU_REGS_R15] = regs->r15;
1360 #endif
1361
1362         vcpu->rip = regs->rip;
1363         kvm_arch_ops->set_rflags(vcpu, regs->rflags);
1364
1365         kvm_arch_ops->decache_regs(vcpu);
1366
1367         vcpu_put(vcpu);
1368
1369         return 0;
1370 }
1371
1372 static void get_segment(struct kvm_vcpu *vcpu,
1373                         struct kvm_segment *var, int seg)
1374 {
1375         return kvm_arch_ops->get_segment(vcpu, var, seg);
1376 }
1377
1378 static int kvm_dev_ioctl_get_sregs(struct kvm *kvm, struct kvm_sregs *sregs)
1379 {
1380         struct kvm_vcpu *vcpu;
1381         struct descriptor_table dt;
1382
1383         if (!valid_vcpu(sregs->vcpu))
1384                 return -EINVAL;
1385         vcpu = vcpu_load(kvm, sregs->vcpu);
1386         if (!vcpu)
1387                 return -ENOENT;
1388
1389         get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
1390         get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
1391         get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
1392         get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
1393         get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
1394         get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
1395
1396         get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
1397         get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
1398
1399         kvm_arch_ops->get_idt(vcpu, &dt);
1400         sregs->idt.limit = dt.limit;
1401         sregs->idt.base = dt.base;
1402         kvm_arch_ops->get_gdt(vcpu, &dt);
1403         sregs->gdt.limit = dt.limit;
1404         sregs->gdt.base = dt.base;
1405
1406         sregs->cr0 = vcpu->cr0;
1407         sregs->cr2 = vcpu->cr2;
1408         sregs->cr3 = vcpu->cr3;
1409         sregs->cr4 = vcpu->cr4;
1410         sregs->cr8 = vcpu->cr8;
1411         sregs->efer = vcpu->shadow_efer;
1412         sregs->apic_base = vcpu->apic_base;
1413
1414         memcpy(sregs->interrupt_bitmap, vcpu->irq_pending,
1415                sizeof sregs->interrupt_bitmap);
1416
1417         vcpu_put(vcpu);
1418
1419         return 0;
1420 }
1421
1422 static void set_segment(struct kvm_vcpu *vcpu,
1423                         struct kvm_segment *var, int seg)
1424 {
1425         return kvm_arch_ops->set_segment(vcpu, var, seg);
1426 }
1427
1428 static int kvm_dev_ioctl_set_sregs(struct kvm *kvm, struct kvm_sregs *sregs)
1429 {
1430         struct kvm_vcpu *vcpu;
1431         int mmu_reset_needed = 0;
1432         int i;
1433         struct descriptor_table dt;
1434
1435         if (!valid_vcpu(sregs->vcpu))
1436                 return -EINVAL;
1437         vcpu = vcpu_load(kvm, sregs->vcpu);
1438         if (!vcpu)
1439                 return -ENOENT;
1440
1441         set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
1442         set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
1443         set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
1444         set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
1445         set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
1446         set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
1447
1448         set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
1449         set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
1450
1451         dt.limit = sregs->idt.limit;
1452         dt.base = sregs->idt.base;
1453         kvm_arch_ops->set_idt(vcpu, &dt);
1454         dt.limit = sregs->gdt.limit;
1455         dt.base = sregs->gdt.base;
1456         kvm_arch_ops->set_gdt(vcpu, &dt);
1457
1458         vcpu->cr2 = sregs->cr2;
1459         mmu_reset_needed |= vcpu->cr3 != sregs->cr3;
1460         vcpu->cr3 = sregs->cr3;
1461
1462         vcpu->cr8 = sregs->cr8;
1463
1464         mmu_reset_needed |= vcpu->shadow_efer != sregs->efer;
1465 #ifdef CONFIG_X86_64
1466         kvm_arch_ops->set_efer(vcpu, sregs->efer);
1467 #endif
1468         vcpu->apic_base = sregs->apic_base;
1469
1470         mmu_reset_needed |= vcpu->cr0 != sregs->cr0;
1471         kvm_arch_ops->set_cr0_no_modeswitch(vcpu, sregs->cr0);
1472
1473         mmu_reset_needed |= vcpu->cr4 != sregs->cr4;
1474         kvm_arch_ops->set_cr4(vcpu, sregs->cr4);
1475
1476         if (mmu_reset_needed)
1477                 kvm_mmu_reset_context(vcpu);
1478
1479         memcpy(vcpu->irq_pending, sregs->interrupt_bitmap,
1480                sizeof vcpu->irq_pending);
1481         vcpu->irq_summary = 0;
1482         for (i = 0; i < NR_IRQ_WORDS; ++i)
1483                 if (vcpu->irq_pending[i])
1484                         __set_bit(i, &vcpu->irq_summary);
1485
1486         vcpu_put(vcpu);
1487
1488         return 0;
1489 }
1490
1491 /*
1492  * List of msr numbers which we expose to userspace through KVM_GET_MSRS
1493  * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
1494  *
1495  * This list is modified at module load time to reflect the
1496  * capabilities of the host cpu.
1497  */
1498 static u32 msrs_to_save[] = {
1499         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
1500         MSR_K6_STAR,
1501 #ifdef CONFIG_X86_64
1502         MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
1503 #endif
1504         MSR_IA32_TIME_STAMP_COUNTER,
1505 };
1506
1507 static unsigned num_msrs_to_save;
1508
1509 static __init void kvm_init_msr_list(void)
1510 {
1511         u32 dummy[2];
1512         unsigned i, j;
1513
1514         for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) {
1515                 if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
1516                         continue;
1517                 if (j < i)
1518                         msrs_to_save[j] = msrs_to_save[i];
1519                 j++;
1520         }
1521         num_msrs_to_save = j;
1522 }
1523
1524 /*
1525  * Adapt set_msr() to msr_io()'s calling convention
1526  */
1527 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
1528 {
1529         return set_msr(vcpu, index, *data);
1530 }
1531
1532 /*
1533  * Read or write a bunch of msrs. All parameters are kernel addresses.
1534  *
1535  * @return number of msrs set successfully.
1536  */
1537 static int __msr_io(struct kvm *kvm, struct kvm_msrs *msrs,
1538                     struct kvm_msr_entry *entries,
1539                     int (*do_msr)(struct kvm_vcpu *vcpu,
1540                                   unsigned index, u64 *data))
1541 {
1542         struct kvm_vcpu *vcpu;
1543         int i;
1544
1545         if (!valid_vcpu(msrs->vcpu))
1546                 return -EINVAL;
1547
1548         vcpu = vcpu_load(kvm, msrs->vcpu);
1549         if (!vcpu)
1550                 return -ENOENT;
1551
1552         for (i = 0; i < msrs->nmsrs; ++i)
1553                 if (do_msr(vcpu, entries[i].index, &entries[i].data))
1554                         break;
1555
1556         vcpu_put(vcpu);
1557
1558         return i;
1559 }
1560
1561 /*
1562  * Read or write a bunch of msrs. Parameters are user addresses.
1563  *
1564  * @return number of msrs set successfully.
1565  */
1566 static int msr_io(struct kvm *kvm, struct kvm_msrs __user *user_msrs,
1567                   int (*do_msr)(struct kvm_vcpu *vcpu,
1568                                 unsigned index, u64 *data),
1569                   int writeback)
1570 {
1571         struct kvm_msrs msrs;
1572         struct kvm_msr_entry *entries;
1573         int r, n;
1574         unsigned size;
1575
1576         r = -EFAULT;
1577         if (copy_from_user(&msrs, user_msrs, sizeof msrs))
1578                 goto out;
1579
1580         r = -E2BIG;
1581         if (msrs.nmsrs >= MAX_IO_MSRS)
1582                 goto out;
1583
1584         r = -ENOMEM;
1585         size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
1586         entries = vmalloc(size);
1587         if (!entries)
1588                 goto out;
1589
1590         r = -EFAULT;
1591         if (copy_from_user(entries, user_msrs->entries, size))
1592                 goto out_free;
1593
1594         r = n = __msr_io(kvm, &msrs, entries, do_msr);
1595         if (r < 0)
1596                 goto out_free;
1597
1598         r = -EFAULT;
1599         if (writeback && copy_to_user(user_msrs->entries, entries, size))
1600                 goto out_free;
1601
1602         r = n;
1603
1604 out_free:
1605         vfree(entries);
1606 out:
1607         return r;
1608 }
1609
1610 /*
1611  * Translate a guest virtual address to a guest physical address.
1612  */
1613 static int kvm_dev_ioctl_translate(struct kvm *kvm, struct kvm_translation *tr)
1614 {
1615         unsigned long vaddr = tr->linear_address;
1616         struct kvm_vcpu *vcpu;
1617         gpa_t gpa;
1618
1619         vcpu = vcpu_load(kvm, tr->vcpu);
1620         if (!vcpu)
1621                 return -ENOENT;
1622         spin_lock(&kvm->lock);
1623         gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr);
1624         tr->physical_address = gpa;
1625         tr->valid = gpa != UNMAPPED_GVA;
1626         tr->writeable = 1;
1627         tr->usermode = 0;
1628         spin_unlock(&kvm->lock);
1629         vcpu_put(vcpu);
1630
1631         return 0;
1632 }
1633
1634 static int kvm_dev_ioctl_interrupt(struct kvm *kvm, struct kvm_interrupt *irq)
1635 {
1636         struct kvm_vcpu *vcpu;
1637
1638         if (!valid_vcpu(irq->vcpu))
1639                 return -EINVAL;
1640         if (irq->irq < 0 || irq->irq >= 256)
1641                 return -EINVAL;
1642         vcpu = vcpu_load(kvm, irq->vcpu);
1643         if (!vcpu)
1644                 return -ENOENT;
1645
1646         set_bit(irq->irq, vcpu->irq_pending);
1647         set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary);
1648
1649         vcpu_put(vcpu);
1650
1651         return 0;
1652 }
1653
1654 static int kvm_dev_ioctl_debug_guest(struct kvm *kvm,
1655                                      struct kvm_debug_guest *dbg)
1656 {
1657         struct kvm_vcpu *vcpu;
1658         int r;
1659
1660         if (!valid_vcpu(dbg->vcpu))
1661                 return -EINVAL;
1662         vcpu = vcpu_load(kvm, dbg->vcpu);
1663         if (!vcpu)
1664                 return -ENOENT;
1665
1666         r = kvm_arch_ops->set_guest_debug(vcpu, dbg);
1667
1668         vcpu_put(vcpu);
1669
1670         return r;
1671 }
1672
1673 static long kvm_dev_ioctl(struct file *filp,
1674                           unsigned int ioctl, unsigned long arg)
1675 {
1676         struct kvm *kvm = filp->private_data;
1677         int r = -EINVAL;
1678
1679         switch (ioctl) {
1680         case KVM_GET_API_VERSION:
1681                 r = KVM_API_VERSION;
1682                 break;
1683         case KVM_CREATE_VCPU: {
1684                 r = kvm_dev_ioctl_create_vcpu(kvm, arg);
1685                 if (r)
1686                         goto out;
1687                 break;
1688         }
1689         case KVM_RUN: {
1690                 struct kvm_run kvm_run;
1691
1692                 r = -EFAULT;
1693                 if (copy_from_user(&kvm_run, (void *)arg, sizeof kvm_run))
1694                         goto out;
1695                 r = kvm_dev_ioctl_run(kvm, &kvm_run);
1696                 if (r < 0)
1697                         goto out;
1698                 r = -EFAULT;
1699                 if (copy_to_user((void *)arg, &kvm_run, sizeof kvm_run))
1700                         goto out;
1701                 r = 0;
1702                 break;
1703         }
1704         case KVM_GET_REGS: {
1705                 struct kvm_regs kvm_regs;
1706
1707                 r = -EFAULT;
1708                 if (copy_from_user(&kvm_regs, (void *)arg, sizeof kvm_regs))
1709                         goto out;
1710                 r = kvm_dev_ioctl_get_regs(kvm, &kvm_regs);
1711                 if (r)
1712                         goto out;
1713                 r = -EFAULT;
1714                 if (copy_to_user((void *)arg, &kvm_regs, sizeof kvm_regs))
1715                         goto out;
1716                 r = 0;
1717                 break;
1718         }
1719         case KVM_SET_REGS: {
1720                 struct kvm_regs kvm_regs;
1721
1722                 r = -EFAULT;
1723                 if (copy_from_user(&kvm_regs, (void *)arg, sizeof kvm_regs))
1724                         goto out;
1725                 r = kvm_dev_ioctl_set_regs(kvm, &kvm_regs);
1726                 if (r)
1727                         goto out;
1728                 r = 0;
1729                 break;
1730         }
1731         case KVM_GET_SREGS: {
1732                 struct kvm_sregs kvm_sregs;
1733
1734                 r = -EFAULT;
1735                 if (copy_from_user(&kvm_sregs, (void *)arg, sizeof kvm_sregs))
1736                         goto out;
1737                 r = kvm_dev_ioctl_get_sregs(kvm, &kvm_sregs);
1738                 if (r)
1739                         goto out;
1740                 r = -EFAULT;
1741                 if (copy_to_user((void *)arg, &kvm_sregs, sizeof kvm_sregs))
1742                         goto out;
1743                 r = 0;
1744                 break;
1745         }
1746         case KVM_SET_SREGS: {
1747                 struct kvm_sregs kvm_sregs;
1748
1749                 r = -EFAULT;
1750                 if (copy_from_user(&kvm_sregs, (void *)arg, sizeof kvm_sregs))
1751                         goto out;
1752                 r = kvm_dev_ioctl_set_sregs(kvm, &kvm_sregs);
1753                 if (r)
1754                         goto out;
1755                 r = 0;
1756                 break;
1757         }
1758         case KVM_TRANSLATE: {
1759                 struct kvm_translation tr;
1760
1761                 r = -EFAULT;
1762                 if (copy_from_user(&tr, (void *)arg, sizeof tr))
1763                         goto out;
1764                 r = kvm_dev_ioctl_translate(kvm, &tr);
1765                 if (r)
1766                         goto out;
1767                 r = -EFAULT;
1768                 if (copy_to_user((void *)arg, &tr, sizeof tr))
1769                         goto out;
1770                 r = 0;
1771                 break;
1772         }
1773         case KVM_INTERRUPT: {
1774                 struct kvm_interrupt irq;
1775
1776                 r = -EFAULT;
1777                 if (copy_from_user(&irq, (void *)arg, sizeof irq))
1778                         goto out;
1779                 r = kvm_dev_ioctl_interrupt(kvm, &irq);
1780                 if (r)
1781                         goto out;
1782                 r = 0;
1783                 break;
1784         }
1785         case KVM_DEBUG_GUEST: {
1786                 struct kvm_debug_guest dbg;
1787
1788                 r = -EFAULT;
1789                 if (copy_from_user(&dbg, (void *)arg, sizeof dbg))
1790                         goto out;
1791                 r = kvm_dev_ioctl_debug_guest(kvm, &dbg);
1792                 if (r)
1793                         goto out;
1794                 r = 0;
1795                 break;
1796         }
1797         case KVM_SET_MEMORY_REGION: {
1798                 struct kvm_memory_region kvm_mem;
1799
1800                 r = -EFAULT;
1801                 if (copy_from_user(&kvm_mem, (void *)arg, sizeof kvm_mem))
1802                         goto out;
1803                 r = kvm_dev_ioctl_set_memory_region(kvm, &kvm_mem);
1804                 if (r)
1805                         goto out;
1806                 break;
1807         }
1808         case KVM_GET_DIRTY_LOG: {
1809                 struct kvm_dirty_log log;
1810
1811                 r = -EFAULT;
1812                 if (copy_from_user(&log, (void *)arg, sizeof log))
1813                         goto out;
1814                 r = kvm_dev_ioctl_get_dirty_log(kvm, &log);
1815                 if (r)
1816                         goto out;
1817                 break;
1818         }
1819         case KVM_GET_MSRS:
1820                 r = msr_io(kvm, (void __user *)arg, get_msr, 1);
1821                 break;
1822         case KVM_SET_MSRS:
1823                 r = msr_io(kvm, (void __user *)arg, do_set_msr, 0);
1824                 break;
1825         case KVM_GET_MSR_INDEX_LIST: {
1826                 struct kvm_msr_list __user *user_msr_list = (void __user *)arg;
1827                 struct kvm_msr_list msr_list;
1828                 unsigned n;
1829
1830                 r = -EFAULT;
1831                 if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
1832                         goto out;
1833                 n = msr_list.nmsrs;
1834                 msr_list.nmsrs = num_msrs_to_save;
1835                 if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
1836                         goto out;
1837                 r = -E2BIG;
1838                 if (n < num_msrs_to_save)
1839                         goto out;
1840                 r = -EFAULT;
1841                 if (copy_to_user(user_msr_list->indices, &msrs_to_save,
1842                                  num_msrs_to_save * sizeof(u32)))
1843                         goto out;
1844                 r = 0;
1845         }
1846         default:
1847                 ;
1848         }
1849 out:
1850         return r;
1851 }
1852
1853 static struct page *kvm_dev_nopage(struct vm_area_struct *vma,
1854                                    unsigned long address,
1855                                    int *type)
1856 {
1857         struct kvm *kvm = vma->vm_file->private_data;
1858         unsigned long pgoff;
1859         struct kvm_memory_slot *slot;
1860         struct page *page;
1861
1862         *type = VM_FAULT_MINOR;
1863         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
1864         slot = gfn_to_memslot(kvm, pgoff);
1865         if (!slot)
1866                 return NOPAGE_SIGBUS;
1867         page = gfn_to_page(slot, pgoff);
1868         if (!page)
1869                 return NOPAGE_SIGBUS;
1870         get_page(page);
1871         return page;
1872 }
1873
1874 static struct vm_operations_struct kvm_dev_vm_ops = {
1875         .nopage = kvm_dev_nopage,
1876 };
1877
1878 static int kvm_dev_mmap(struct file *file, struct vm_area_struct *vma)
1879 {
1880         vma->vm_ops = &kvm_dev_vm_ops;
1881         return 0;
1882 }
1883
1884 static struct file_operations kvm_chardev_ops = {
1885         .open           = kvm_dev_open,
1886         .release        = kvm_dev_release,
1887         .unlocked_ioctl = kvm_dev_ioctl,
1888         .compat_ioctl   = kvm_dev_ioctl,
1889         .mmap           = kvm_dev_mmap,
1890 };
1891
1892 static struct miscdevice kvm_dev = {
1893         MISC_DYNAMIC_MINOR,
1894         "kvm",
1895         &kvm_chardev_ops,
1896 };
1897
1898 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
1899                        void *v)
1900 {
1901         if (val == SYS_RESTART) {
1902                 /*
1903                  * Some (well, at least mine) BIOSes hang on reboot if
1904                  * in vmx root mode.
1905                  */
1906                 printk(KERN_INFO "kvm: exiting hardware virtualization\n");
1907                 on_each_cpu(kvm_arch_ops->hardware_disable, 0, 0, 1);
1908         }
1909         return NOTIFY_OK;
1910 }
1911
1912 static struct notifier_block kvm_reboot_notifier = {
1913         .notifier_call = kvm_reboot,
1914         .priority = 0,
1915 };
1916
1917 static __init void kvm_init_debug(void)
1918 {
1919         struct kvm_stats_debugfs_item *p;
1920
1921         debugfs_dir = debugfs_create_dir("kvm", 0);
1922         for (p = debugfs_entries; p->name; ++p)
1923                 p->dentry = debugfs_create_u32(p->name, 0444, debugfs_dir,
1924                                                p->data);
1925 }
1926
1927 static void kvm_exit_debug(void)
1928 {
1929         struct kvm_stats_debugfs_item *p;
1930
1931         for (p = debugfs_entries; p->name; ++p)
1932                 debugfs_remove(p->dentry);
1933         debugfs_remove(debugfs_dir);
1934 }
1935
1936 hpa_t bad_page_address;
1937
1938 int kvm_init_arch(struct kvm_arch_ops *ops, struct module *module)
1939 {
1940         int r;
1941
1942         if (kvm_arch_ops) {
1943                 printk(KERN_ERR "kvm: already loaded the other module\n");
1944                 return -EEXIST;
1945         }
1946
1947         kvm_arch_ops = ops;
1948
1949         if (!kvm_arch_ops->cpu_has_kvm_support()) {
1950                 printk(KERN_ERR "kvm: no hardware support\n");
1951                 return -EOPNOTSUPP;
1952         }
1953         if (kvm_arch_ops->disabled_by_bios()) {
1954                 printk(KERN_ERR "kvm: disabled by bios\n");
1955                 return -EOPNOTSUPP;
1956         }
1957
1958         r = kvm_arch_ops->hardware_setup();
1959         if (r < 0)
1960             return r;
1961
1962         on_each_cpu(kvm_arch_ops->hardware_enable, 0, 0, 1);
1963         register_reboot_notifier(&kvm_reboot_notifier);
1964
1965         kvm_chardev_ops.owner = module;
1966
1967         r = misc_register(&kvm_dev);
1968         if (r) {
1969                 printk (KERN_ERR "kvm: misc device register failed\n");
1970                 goto out_free;
1971         }
1972
1973         return r;
1974
1975 out_free:
1976         unregister_reboot_notifier(&kvm_reboot_notifier);
1977         on_each_cpu(kvm_arch_ops->hardware_disable, 0, 0, 1);
1978         kvm_arch_ops->hardware_unsetup();
1979         return r;
1980 }
1981
1982 void kvm_exit_arch(void)
1983 {
1984         misc_deregister(&kvm_dev);
1985
1986         unregister_reboot_notifier(&kvm_reboot_notifier);
1987         on_each_cpu(kvm_arch_ops->hardware_disable, 0, 0, 1);
1988         kvm_arch_ops->hardware_unsetup();
1989         kvm_arch_ops = NULL;
1990 }
1991
1992 static __init int kvm_init(void)
1993 {
1994         static struct page *bad_page;
1995         int r = 0;
1996
1997         kvm_init_debug();
1998
1999         kvm_init_msr_list();
2000
2001         if ((bad_page = alloc_page(GFP_KERNEL)) == NULL) {
2002                 r = -ENOMEM;
2003                 goto out;
2004         }
2005
2006         bad_page_address = page_to_pfn(bad_page) << PAGE_SHIFT;
2007         memset(__va(bad_page_address), 0, PAGE_SIZE);
2008
2009         return r;
2010
2011 out:
2012         kvm_exit_debug();
2013         return r;
2014 }
2015
2016 static __exit void kvm_exit(void)
2017 {
2018         kvm_exit_debug();
2019         __free_page(pfn_to_page(bad_page_address >> PAGE_SHIFT));
2020 }
2021
2022 module_init(kvm_init)
2023 module_exit(kvm_exit)
2024
2025 EXPORT_SYMBOL_GPL(kvm_init_arch);
2026 EXPORT_SYMBOL_GPL(kvm_exit_arch);