arch/x86/kvm/mmu.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * MMU support
   8  *
   9  * Copyright (C) 2006 Qumranet, Inc.
  10  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
  11  *
  12  * Authors:
  13  *   Yaniv Kamay  <yaniv@qumranet.com>
  14  *   Avi Kivity   <avi@qumranet.com>
  15  *
  16  * This work is licensed under the terms of the GNU GPL, version 2.  See
  17  * the COPYING file in the top-level directory.
  18  *
  19  */
  20
  21 #include "mmu.h"
  22 #include "x86.h"
  23 #include "kvm_cache_regs.h"
  24
  25 #include <linux/kvm_host.h>
  26 #include <linux/types.h>
  27 #include <linux/string.h>
  28 #include <linux/mm.h>
  29 #include <linux/highmem.h>
  30 #include <linux/module.h>
  31 #include <linux/swap.h>
  32 #include <linux/hugetlb.h>
  33 #include <linux/compiler.h>
  34 #include <linux/srcu.h>
  35 #include <linux/slab.h>
  36 #include <linux/uaccess.h>
  37
  38 #include <asm/page.h>
  39 #include <asm/cmpxchg.h>
  40 #include <asm/io.h>
  41 #include <asm/vmx.h>
  42
  43 /*
  44  * When setting this variable to true it enables Two-Dimensional-Paging
  45  * where the hardware walks 2 page tables:
  46  * 1. the guest-virtual to guest-physical
  47  * 2. while doing 1. it walks guest-physical to host-physical
  48  * If the hardware supports that we don't need to do shadow paging.
  49  */
  50 bool tdp_enabled = false;
  51
  52 enum {
  53         AUDIT_PRE_PAGE_FAULT,
  54         AUDIT_POST_PAGE_FAULT,
  55         AUDIT_PRE_PTE_WRITE,
  56         AUDIT_POST_PTE_WRITE,
  57         AUDIT_PRE_SYNC,
  58         AUDIT_POST_SYNC
  59 };
  60
  61 char *audit_point_name[] = {
  62         "pre page fault",
  63         "post page fault",
  64         "pre pte write",
  65         "post pte write",
  66         "pre sync",
  67         "post sync"
  68 };
  69
  70 #undef MMU_DEBUG
  71
  72 #ifdef MMU_DEBUG
  73
  74 #define pgprintk(x...) do { if (dbg) printk(x); } while (0)
  75 #define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
  76
  77 #else
  78
  79 #define pgprintk(x...) do { } while (0)
  80 #define rmap_printk(x...) do { } while (0)
  81
  82 #endif
  83
  84 #ifdef MMU_DEBUG
  85 static int dbg = 0;
  86 module_param(dbg, bool, 0644);
  87 #endif
  88
  89 static int oos_shadow = 1;
  90 module_param(oos_shadow, bool, 0644);
  91
  92 #ifndef MMU_DEBUG
  93 #define ASSERT(x) do { } while (0)
  94 #else
  95 #define ASSERT(x)                                                       \
  96         if (!(x)) {                                                     \
  97                 printk(KERN_WARNING "assertion failed %s:%d: %s\n",     \
  98                        __FILE__, __LINE__, #x);                         \
  99         }
 100 #endif
 101
 102 #define PTE_PREFETCH_NUM                8
 103
 104 #define PT_FIRST_AVAIL_BITS_SHIFT 9
 105 #define PT64_SECOND_AVAIL_BITS_SHIFT 52
 106
 107 #define PT64_LEVEL_BITS 9
 108
 109 #define PT64_LEVEL_SHIFT(level) \
 110                 (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
 111
 112 #define PT64_LEVEL_MASK(level) \
 113                 (((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level))
 114
 115 #define PT64_INDEX(address, level)\
 116         (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
 117
 118
 119 #define PT32_LEVEL_BITS 10
 120
 121 #define PT32_LEVEL_SHIFT(level) \
 122                 (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
 123
 124 #define PT32_LEVEL_MASK(level) \
 125                 (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))
 126 #define PT32_LVL_OFFSET_MASK(level) \
 127         (PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
 128                                                 * PT32_LEVEL_BITS))) - 1))
 129
 130 #define PT32_INDEX(address, level)\
 131         (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
 132
 133
 134 #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
 135 #define PT64_DIR_BASE_ADDR_MASK \
 136         (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
 137 #define PT64_LVL_ADDR_MASK(level) \
 138         (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
 139                                                 * PT64_LEVEL_BITS))) - 1))
 140 #define PT64_LVL_OFFSET_MASK(level) \
 141         (PT64_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
 142                                                 * PT64_LEVEL_BITS))) - 1))
 143
 144 #define PT32_BASE_ADDR_MASK PAGE_MASK
 145 #define PT32_DIR_BASE_ADDR_MASK \
 146         (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
 147 #define PT32_LVL_ADDR_MASK(level) \
 148         (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
 149                                             * PT32_LEVEL_BITS))) - 1))
 150
 151 #define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK \
 152                         | PT64_NX_MASK)
 153
 154 #define RMAP_EXT 4
 155
 156 #define ACC_EXEC_MASK    1
 157 #define ACC_WRITE_MASK   PT_WRITABLE_MASK
 158 #define ACC_USER_MASK    PT_USER_MASK
 159 #define ACC_ALL          (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
 160
 161 #include <trace/events/kvm.h>
 162
 163 #define CREATE_TRACE_POINTS
 164 #include "mmutrace.h"
 165
 166 #define SPTE_HOST_WRITEABLE (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
 167
 168 #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)
 169
 170 struct kvm_rmap_desc {
 171         u64 *sptes[RMAP_EXT];
 172         struct kvm_rmap_desc *more;
 173 };
 174
 175 struct kvm_shadow_walk_iterator {
 176         u64 addr;
 177         hpa_t shadow_addr;
 178         int level;
 179         u64 *sptep;
 180         unsigned index;
 181 };
 182
 183 #define for_each_shadow_entry(_vcpu, _addr, _walker)    \
 184         for (shadow_walk_init(&(_walker), _vcpu, _addr);        \
 185              shadow_walk_okay(&(_walker));                      \
 186              shadow_walk_next(&(_walker)))
 187
 188 typedef void (*mmu_parent_walk_fn) (struct kvm_mmu_page *sp, u64 *spte);
 189
 190 static struct kmem_cache *pte_chain_cache;
 191 static struct kmem_cache *rmap_desc_cache;
 192 static struct kmem_cache *mmu_page_header_cache;
 193 static struct percpu_counter kvm_total_used_mmu_pages;
 194
 195 static u64 __read_mostly shadow_trap_nonpresent_pte;
 196 static u64 __read_mostly shadow_notrap_nonpresent_pte;
 197 static u64 __read_mostly shadow_base_present_pte;
 198 static u64 __read_mostly shadow_nx_mask;
 199 static u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
 200 static u64 __read_mostly shadow_user_mask;
 201 static u64 __read_mostly shadow_accessed_mask;
 202 static u64 __read_mostly shadow_dirty_mask;
 203
 204 static inline u64 rsvd_bits(int s, int e)
 205 {
 206         return ((1ULL << (e - s + 1)) - 1) << s;
 207 }
 208
 209 void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte)
 210 {
 211         shadow_trap_nonpresent_pte = trap_pte;
 212         shadow_notrap_nonpresent_pte = notrap_pte;
 213 }
 214 EXPORT_SYMBOL_GPL(kvm_mmu_set_nonpresent_ptes);
 215
 216 void kvm_mmu_set_base_ptes(u64 base_pte)
 217 {
 218         shadow_base_present_pte = base_pte;
 219 }
 220 EXPORT_SYMBOL_GPL(kvm_mmu_set_base_ptes);
 221
 222 void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
 223                 u64 dirty_mask, u64 nx_mask, u64 x_mask)
 224 {
 225         shadow_user_mask = user_mask;
 226         shadow_accessed_mask = accessed_mask;
 227         shadow_dirty_mask = dirty_mask;
 228         shadow_nx_mask = nx_mask;
 229         shadow_x_mask = x_mask;
 230 }
 231 EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);
 232
 233 static bool is_write_protection(struct kvm_vcpu *vcpu)
 234 {
 235         return kvm_read_cr0_bits(vcpu, X86_CR0_WP);
 236 }
 237
 238 static int is_cpuid_PSE36(void)
 239 {
 240         return 1;
 241 }
 242
 243 static int is_nx(struct kvm_vcpu *vcpu)
 244 {
 245         return vcpu->arch.efer & EFER_NX;
 246 }
 247
 248 static int is_shadow_present_pte(u64 pte)
 249 {
 250         return pte != shadow_trap_nonpresent_pte
 251                 && pte != shadow_notrap_nonpresent_pte;
 252 }
 253
 254 static int is_large_pte(u64 pte)
 255 {
 256         return pte & PT_PAGE_SIZE_MASK;
 257 }
 258
 259 static int is_writable_pte(unsigned long pte)
 260 {
 261         return pte & PT_WRITABLE_MASK;
 262 }
 263
 264 static int is_dirty_gpte(unsigned long pte)
 265 {
 266         return pte & PT_DIRTY_MASK;
 267 }
 268
 269 static int is_rmap_spte(u64 pte)
 270 {
 271         return is_shadow_present_pte(pte);
 272 }
 273
 274 static int is_last_spte(u64 pte, int level)
 275 {
 276         if (level == PT_PAGE_TABLE_LEVEL)
 277                 return 1;
 278         if (is_large_pte(pte))
 279                 return 1;
 280         return 0;
 281 }
 282
 283 static pfn_t spte_to_pfn(u64 pte)
 284 {
 285         return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
 286 }
 287
 288 static gfn_t pse36_gfn_delta(u32 gpte)
 289 {
 290         int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;
 291
 292         return (gpte & PT32_DIR_PSE36_MASK) << shift;
 293 }
 294
 295 static void __set_spte(u64 *sptep, u64 spte)
 296 {
 297         set_64bit(sptep, spte);
 298 }
 299
 300 static u64 __xchg_spte(u64 *sptep, u64 new_spte)
 301 {
 302 #ifdef CONFIG_X86_64
 303         return xchg(sptep, new_spte);
 304 #else
 305         u64 old_spte;
 306
 307         do {
 308                 old_spte = *sptep;
 309         } while (cmpxchg64(sptep, old_spte, new_spte) != old_spte);
 310
 311         return old_spte;
 312 #endif
 313 }
 314
 315 static bool spte_has_volatile_bits(u64 spte)
 316 {
 317         if (!shadow_accessed_mask)
 318                 return false;
 319
 320         if (!is_shadow_present_pte(spte))
 321                 return false;
 322
 323         if ((spte & shadow_accessed_mask) &&
 324               (!is_writable_pte(spte) || (spte & shadow_dirty_mask)))
 325                 return false;
 326
 327         return true;
 328 }
 329
 330 static bool spte_is_bit_cleared(u64 old_spte, u64 new_spte, u64 bit_mask)
 331 {
 332         return (old_spte & bit_mask) && !(new_spte & bit_mask);
 333 }
 334
 335 static void update_spte(u64 *sptep, u64 new_spte)
 336 {
 337         u64 mask, old_spte = *sptep;
 338
 339         WARN_ON(!is_rmap_spte(new_spte));
 340
 341         new_spte |= old_spte & shadow_dirty_mask;
 342
 343         mask = shadow_accessed_mask;
 344         if (is_writable_pte(old_spte))
 345                 mask |= shadow_dirty_mask;
 346
 347         if (!spte_has_volatile_bits(old_spte) || (new_spte & mask) == mask)
 348                 __set_spte(sptep, new_spte);
 349         else
 350                 old_spte = __xchg_spte(sptep, new_spte);
 351
 352         if (!shadow_accessed_mask)
 353                 return;
 354
 355         if (spte_is_bit_cleared(old_spte, new_spte, shadow_accessed_mask))
 356                 kvm_set_pfn_accessed(spte_to_pfn(old_spte));
 357         if (spte_is_bit_cleared(old_spte, new_spte, shadow_dirty_mask))
 358                 kvm_set_pfn_dirty(spte_to_pfn(old_spte));
 359 }
 360
 361 static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
 362                                   struct kmem_cache *base_cache, int min)
 363 {
 364         void *obj;
 365
 366         if (cache->nobjs >= min)
 367                 return 0;
 368         while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
 369                 obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
 370                 if (!obj)
 371                         return -ENOMEM;
 372                 cache->objects[cache->nobjs++] = obj;
 373         }
 374         return 0;
 375 }
 376
 377 static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc,
 378                                   struct kmem_cache *cache)
 379 {
 380         while (mc->nobjs)
 381                 kmem_cache_free(cache, mc->objects[--mc->nobjs]);
 382 }
 383
 384 static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
 385                                        int min)
 386 {
 387         struct page *page;
 388
 389         if (cache->nobjs >= min)
 390                 return 0;
 391         while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
 392                 page = alloc_page(GFP_KERNEL);
 393                 if (!page)
 394                         return -ENOMEM;
 395                 cache->objects[cache->nobjs++] = page_address(page);
 396         }
 397         return 0;
 398 }
 399
 400 static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
 401 {
 402         while (mc->nobjs)
 403                 free_page((unsigned long)mc->objects[--mc->nobjs]);
 404 }
 405
 406 static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
 407 {
 408         int r;
 409
 410         r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_chain_cache,
 411                                    pte_chain_cache, 4);
 412         if (r)
 413                 goto out;
 414         r = mmu_topup_memory_cache(&vcpu->arch.mmu_rmap_desc_cache,
 415                                    rmap_desc_cache, 4 + PTE_PREFETCH_NUM);
 416         if (r)
 417                 goto out;
 418         r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
 419         if (r)
 420                 goto out;
 421         r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
 422                                    mmu_page_header_cache, 4);
 423 out:
 424         return r;
 425 }
 426
 427 static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
 428 {
 429         mmu_free_memory_cache(&vcpu->arch.mmu_pte_chain_cache, pte_chain_cache);
 430         mmu_free_memory_cache(&vcpu->arch.mmu_rmap_desc_cache, rmap_desc_cache);
 431         mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
 432         mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache,
 433                                 mmu_page_header_cache);
 434 }
 435
 436 static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
 437                                     size_t size)
 438 {
 439         void *p;
 440
 441         BUG_ON(!mc->nobjs);
 442         p = mc->objects[--mc->nobjs];
 443         return p;
 444 }
 445
 446 static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu)
 447 {
 448         return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_chain_cache,
 449                                       sizeof(struct kvm_pte_chain));
 450 }
 451
 452 static void mmu_free_pte_chain(struct kvm_pte_chain *pc)
 453 {
 454         kmem_cache_free(pte_chain_cache, pc);
 455 }
 456
 457 static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu)
 458 {
 459         return mmu_memory_cache_alloc(&vcpu->arch.mmu_rmap_desc_cache,
 460                                       sizeof(struct kvm_rmap_desc));
 461 }
 462
 463 static void mmu_free_rmap_desc(struct kvm_rmap_desc *rd)
 464 {
 465         kmem_cache_free(rmap_desc_cache, rd);
 466 }
 467
 468 static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index)
 469 {
 470         if (!sp->role.direct)
 471                 return sp->gfns[index];
 472
 473         return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS));
 474 }
 475
 476 static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn)
 477 {
 478         if (sp->role.direct)
 479                 BUG_ON(gfn != kvm_mmu_page_get_gfn(sp, index));
 480         else
 481                 sp->gfns[index] = gfn;
 482 }
 483
 484 /*
 485  * Return the pointer to the largepage write count for a given
 486  * gfn, handling slots that are not large page aligned.
 487  */
 488 static int *slot_largepage_idx(gfn_t gfn,
 489                                struct kvm_memory_slot *slot,
 490                                int level)
 491 {
 492         unsigned long idx;
 493
 494         idx = (gfn >> KVM_HPAGE_GFN_SHIFT(level)) -
 495               (slot->base_gfn >> KVM_HPAGE_GFN_SHIFT(level));
 496         return &slot->lpage_info[level - 2][idx].write_count;
 497 }
 498
 499 static void account_shadowed(struct kvm *kvm, gfn_t gfn)
 500 {
 501         struct kvm_memory_slot *slot;
 502         int *write_count;
 503         int i;
 504
 505         slot = gfn_to_memslot(kvm, gfn);
 506         for (i = PT_DIRECTORY_LEVEL;
 507              i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
 508                 write_count   = slot_largepage_idx(gfn, slot, i);
 509                 *write_count += 1;
 510         }
 511 }
 512
 513 static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn)
 514 {
 515         struct kvm_memory_slot *slot;
 516         int *write_count;
 517         int i;
 518
 519         slot = gfn_to_memslot(kvm, gfn);
 520         for (i = PT_DIRECTORY_LEVEL;
 521              i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
 522                 write_count   = slot_largepage_idx(gfn, slot, i);
 523                 *write_count -= 1;
 524                 WARN_ON(*write_count < 0);
 525         }
 526 }
 527
 528 static int has_wrprotected_page(struct kvm *kvm,
 529                                 gfn_t gfn,
 530                                 int level)
 531 {
 532         struct kvm_memory_slot *slot;
 533         int *largepage_idx;
 534
 535         slot = gfn_to_memslot(kvm, gfn);
 536         if (slot) {
 537                 largepage_idx = slot_largepage_idx(gfn, slot, level);
 538                 return *largepage_idx;
 539         }
 540
 541         return 1;
 542 }
 543
 544 static int host_mapping_level(struct kvm *kvm, gfn_t gfn)
 545 {
 546         unsigned long page_size;
 547         int i, ret = 0;
 548
 549         page_size = kvm_host_page_size(kvm, gfn);
 550
 551         for (i = PT_PAGE_TABLE_LEVEL;
 552              i < (PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES); ++i) {
 553                 if (page_size >= KVM_HPAGE_SIZE(i))
 554                         ret = i;
 555                 else
 556                         break;
 557         }
 558
 559         return ret;
 560 }
 561
 562 static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn)
 563 {
 564         struct kvm_memory_slot *slot;
 565         int host_level, level, max_level;
 566
 567         slot = gfn_to_memslot(vcpu->kvm, large_gfn);
 568         if (slot && slot->dirty_bitmap)
 569                 return PT_PAGE_TABLE_LEVEL;
 570
 571         host_level = host_mapping_level(vcpu->kvm, large_gfn);
 572
 573         if (host_level == PT_PAGE_TABLE_LEVEL)
 574                 return host_level;
 575
 576         max_level = kvm_x86_ops->get_lpage_level() < host_level ?
 577                 kvm_x86_ops->get_lpage_level() : host_level;
 578
 579         for (level = PT_DIRECTORY_LEVEL; level <= max_level; ++level)
 580                 if (has_wrprotected_page(vcpu->kvm, large_gfn, level))
 581                         break;
 582
 583         return level - 1;
 584 }
 585
 586 /*
 587  * Take gfn and return the reverse mapping to it.
 588  */
 589
 590 static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int level)
 591 {
 592         struct kvm_memory_slot *slot;
 593         unsigned long idx;
 594
 595         slot = gfn_to_memslot(kvm, gfn);
 596         if (likely(level == PT_PAGE_TABLE_LEVEL))
 597                 return &slot->rmap[gfn - slot->base_gfn];
 598
 599         idx = (gfn >> KVM_HPAGE_GFN_SHIFT(level)) -
 600                 (slot->base_gfn >> KVM_HPAGE_GFN_SHIFT(level));
 601
 602         return &slot->lpage_info[level - 2][idx].rmap_pde;
 603 }
 604
 605 /*
 606  * Reverse mapping data structures:
 607  *
 608  * If rmapp bit zero is zero, then rmapp point to the shadw page table entry
 609  * that points to page_address(page).
 610  *
 611  * If rmapp bit zero is one, (then rmap & ~1) points to a struct kvm_rmap_desc
 612  * containing more mappings.
 613  *
 614  * Returns the number of rmap entries before the spte was added or zero if
 615  * the spte was not added.
 616  *
 617  */
 618 static int rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
 619 {
 620         struct kvm_mmu_page *sp;
 621         struct kvm_rmap_desc *desc;
 622         unsigned long *rmapp;
 623         int i, count = 0;
 624
 625         if (!is_rmap_spte(*spte))
 626                 return count;
 627         sp = page_header(__pa(spte));
 628         kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn);
 629         rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
 630         if (!*rmapp) {
 631                 rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
 632                 *rmapp = (unsigned long)spte;
 633         } else if (!(*rmapp & 1)) {
 634                 rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
 635                 desc = mmu_alloc_rmap_desc(vcpu);
 636                 desc->sptes[0] = (u64 *)*rmapp;
 637                 desc->sptes[1] = spte;
 638                 *rmapp = (unsigned long)desc | 1;
 639                 ++count;
 640         } else {
 641                 rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
 642                 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
 643                 while (desc->sptes[RMAP_EXT-1] && desc->more) {
 644                         desc = desc->more;
 645                         count += RMAP_EXT;
 646                 }
 647                 if (desc->sptes[RMAP_EXT-1]) {
 648                         desc->more = mmu_alloc_rmap_desc(vcpu);
 649                         desc = desc->more;
 650                 }
 651                 for (i = 0; desc->sptes[i]; ++i)
 652                         ++count;
 653                 desc->sptes[i] = spte;
 654         }
 655         return count;
 656 }
 657
 658 static void rmap_desc_remove_entry(unsigned long *rmapp,
 659                                    struct kvm_rmap_desc *desc,
 660                                    int i,
 661                                    struct kvm_rmap_desc *prev_desc)
 662 {
 663         int j;
 664
 665         for (j = RMAP_EXT - 1; !desc->sptes[j] && j > i; --j)
 666                 ;
 667         desc->sptes[i] = desc->sptes[j];
 668         desc->sptes[j] = NULL;
 669         if (j != 0)
 670                 return;
 671         if (!prev_desc && !desc->more)
 672                 *rmapp = (unsigned long)desc->sptes[0];
 673         else
 674                 if (prev_desc)
 675                         prev_desc->more = desc->more;
 676                 else
 677                         *rmapp = (unsigned long)desc->more | 1;
 678         mmu_free_rmap_desc(desc);
 679 }
 680
 681 static void rmap_remove(struct kvm *kvm, u64 *spte)
 682 {
 683         struct kvm_rmap_desc *desc;
 684         struct kvm_rmap_desc *prev_desc;
 685         struct kvm_mmu_page *sp;
 686         gfn_t gfn;
 687         unsigned long *rmapp;
 688         int i;
 689
 690         sp = page_header(__pa(spte));
 691         gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt);
 692         rmapp = gfn_to_rmap(kvm, gfn, sp->role.level);
 693         if (!*rmapp) {
 694                 printk(KERN_ERR "rmap_remove: %p 0->BUG\n", spte);
 695                 BUG();
 696         } else if (!(*rmapp & 1)) {
 697                 rmap_printk("rmap_remove:  %p 1->0\n", spte);
 698                 if ((u64 *)*rmapp != spte) {
 699                         printk(KERN_ERR "rmap_remove:  %p 1->BUG\n", spte);
 700                         BUG();
 701                 }
 702                 *rmapp = 0;
 703         } else {
 704                 rmap_printk("rmap_remove:  %p many->many\n", spte);
 705                 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
 706                 prev_desc = NULL;
 707                 while (desc) {
 708                         for (i = 0; i < RMAP_EXT && desc->sptes[i]; ++i)
 709                                 if (desc->sptes[i] == spte) {
 710                                         rmap_desc_remove_entry(rmapp,
 711                                                                desc, i,
 712                                                                prev_desc);
 713                                         return;
 714                                 }
 715                         prev_desc = desc;
 716                         desc = desc->more;
 717                 }
 718                 pr_err("rmap_remove: %p many->many\n", spte);
 719                 BUG();
 720         }
 721 }
 722
 723 static void set_spte_track_bits(u64 *sptep, u64 new_spte)
 724 {
 725         pfn_t pfn;
 726         u64 old_spte = *sptep;
 727
 728         if (!spte_has_volatile_bits(old_spte))
 729                 __set_spte(sptep, new_spte);
 730         else
 731                 old_spte = __xchg_spte(sptep, new_spte);
 732
 733         if (!is_rmap_spte(old_spte))
 734                 return;
 735
 736         pfn = spte_to_pfn(old_spte);
 737         if (!shadow_accessed_mask || old_spte & shadow_accessed_mask)
 738                 kvm_set_pfn_accessed(pfn);
 739         if (!shadow_dirty_mask || (old_spte & shadow_dirty_mask))
 740                 kvm_set_pfn_dirty(pfn);
 741 }
 742
 743 static void drop_spte(struct kvm *kvm, u64 *sptep, u64 new_spte)
 744 {
 745         set_spte_track_bits(sptep, new_spte);
 746         rmap_remove(kvm, sptep);
 747 }
 748
 749 static u64 *rmap_next(struct kvm *kvm, unsigned long *rmapp, u64 *spte)
 750 {
 751         struct kvm_rmap_desc *desc;
 752         u64 *prev_spte;
 753         int i;
 754
 755         if (!*rmapp)
 756                 return NULL;
 757         else if (!(*rmapp & 1)) {
 758                 if (!spte)
 759                         return (u64 *)*rmapp;
 760                 return NULL;
 761         }
 762         desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
 763         prev_spte = NULL;
 764         while (desc) {
 765                 for (i = 0; i < RMAP_EXT && desc->sptes[i]; ++i) {
 766                         if (prev_spte == spte)
 767                                 return desc->sptes[i];
 768                         prev_spte = desc->sptes[i];
 769                 }
 770                 desc = desc->more;
 771         }
 772         return NULL;
 773 }
 774
 775 static int rmap_write_protect(struct kvm *kvm, u64 gfn)
 776 {
 777         unsigned long *rmapp;
 778         u64 *spte;
 779         int i, write_protected = 0;
 780
 781         rmapp = gfn_to_rmap(kvm, gfn, PT_PAGE_TABLE_LEVEL);
 782
 783         spte = rmap_next(kvm, rmapp, NULL);
 784         while (spte) {
 785                 BUG_ON(!spte);
 786                 BUG_ON(!(*spte & PT_PRESENT_MASK));
 787                 rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
 788                 if (is_writable_pte(*spte)) {
 789                         update_spte(spte, *spte & ~PT_WRITABLE_MASK);
 790                         write_protected = 1;
 791                 }
 792                 spte = rmap_next(kvm, rmapp, spte);
 793         }
 794
 795         /* check for huge page mappings */
 796         for (i = PT_DIRECTORY_LEVEL;
 797              i < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++i) {
 798                 rmapp = gfn_to_rmap(kvm, gfn, i);
 799                 spte = rmap_next(kvm, rmapp, NULL);
 800                 while (spte) {
 801                         BUG_ON(!spte);
 802                         BUG_ON(!(*spte & PT_PRESENT_MASK));
 803                         BUG_ON((*spte & (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK)) != (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK));
 804                         pgprintk("rmap_write_protect(large): spte %p %llx %lld\n", spte, *spte, gfn);
 805                         if (is_writable_pte(*spte)) {
 806                                 drop_spte(kvm, spte,
 807                                           shadow_trap_nonpresent_pte);
 808                                 --kvm->stat.lpages;
 809                                 spte = NULL;
 810                                 write_protected = 1;
 811                         }
 812                         spte = rmap_next(kvm, rmapp, spte);
 813                 }
 814         }
 815
 816         return write_protected;
 817 }
 818
 819 static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
 820                            unsigned long data)
 821 {
 822         u64 *spte;
 823         int need_tlb_flush = 0;
 824
 825         while ((spte = rmap_next(kvm, rmapp, NULL))) {
 826                 BUG_ON(!(*spte & PT_PRESENT_MASK));
 827                 rmap_printk("kvm_rmap_unmap_hva: spte %p %llx\n", spte, *spte);
 828                 drop_spte(kvm, spte, shadow_trap_nonpresent_pte);
 829                 need_tlb_flush = 1;
 830         }
 831         return need_tlb_flush;
 832 }
 833
 834 static int kvm_set_pte_rmapp(struct kvm *kvm, unsigned long *rmapp,
 835                              unsigned long data)
 836 {
 837         int need_flush = 0;
 838         u64 *spte, new_spte;
 839         pte_t *ptep = (pte_t *)data;
 840         pfn_t new_pfn;
 841
 842         WARN_ON(pte_huge(*ptep));
 843         new_pfn = pte_pfn(*ptep);
 844         spte = rmap_next(kvm, rmapp, NULL);
 845         while (spte) {
 846                 BUG_ON(!is_shadow_present_pte(*spte));
 847                 rmap_printk("kvm_set_pte_rmapp: spte %p %llx\n", spte, *spte);
 848                 need_flush = 1;
 849                 if (pte_write(*ptep)) {
 850                         drop_spte(kvm, spte, shadow_trap_nonpresent_pte);
 851                         spte = rmap_next(kvm, rmapp, NULL);
 852                 } else {
 853                         new_spte = *spte &~ (PT64_BASE_ADDR_MASK);
 854                         new_spte |= (u64)new_pfn << PAGE_SHIFT;
 855
 856                         new_spte &= ~PT_WRITABLE_MASK;
 857                         new_spte &= ~SPTE_HOST_WRITEABLE;
 858                         new_spte &= ~shadow_accessed_mask;
 859                         set_spte_track_bits(spte, new_spte);
 860                         spte = rmap_next(kvm, rmapp, spte);
 861                 }
 862         }
 863         if (need_flush)
 864                 kvm_flush_remote_tlbs(kvm);
 865
 866         return 0;
 867 }
 868
 869 static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
 870                           unsigned long data,
 871                           int (*handler)(struct kvm *kvm, unsigned long *rmapp,
 872                                          unsigned long data))
 873 {
 874         int i, j;
 875         int ret;
 876         int retval = 0;
 877         struct kvm_memslots *slots;
 878
 879         slots = kvm_memslots(kvm);
 880
 881         for (i = 0; i < slots->nmemslots; i++) {
 882                 struct kvm_memory_slot *memslot = &slots->memslots[i];
 883                 unsigned long start = memslot->userspace_addr;
 884                 unsigned long end;
 885
 886                 end = start + (memslot->npages << PAGE_SHIFT);
 887                 if (hva >= start && hva < end) {
 888                         gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT;
 889
 890                         ret = handler(kvm, &memslot->rmap[gfn_offset], data);
 891
 892                         for (j = 0; j < KVM_NR_PAGE_SIZES - 1; ++j) {
 893                                 unsigned long idx;
 894                                 int sh;
 895
 896                                 sh = KVM_HPAGE_GFN_SHIFT(PT_DIRECTORY_LEVEL+j);
 897                                 idx = ((memslot->base_gfn+gfn_offset) >> sh) -
 898                                         (memslot->base_gfn >> sh);
 899                                 ret |= handler(kvm,
 900                                         &memslot->lpage_info[j][idx].rmap_pde,
 901                                         data);
 902                         }
 903                         trace_kvm_age_page(hva, memslot, ret);
 904                         retval |= ret;
 905                 }
 906         }
 907
 908         return retval;
 909 }
 910
 911 int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
 912 {
 913         return kvm_handle_hva(kvm, hva, 0, kvm_unmap_rmapp);
 914 }
 915
 916 void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
 917 {
 918         kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);
 919 }
 920
 921 static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
 922                          unsigned long data)
 923 {
 924         u64 *spte;
 925         int young = 0;
 926
 927         /*
 928          * Emulate the accessed bit for EPT, by checking if this page has
 929          * an EPT mapping, and clearing it if it does. On the next access,
 930          * a new EPT mapping will be established.
 931          * This has some overhead, but not as much as the cost of swapping
 932          * out actively used pages or breaking up actively used hugepages.
 933          */
 934         if (!shadow_accessed_mask)
 935                 return kvm_unmap_rmapp(kvm, rmapp, data);
 936
 937         spte = rmap_next(kvm, rmapp, NULL);
 938         while (spte) {
 939                 int _young;
 940                 u64 _spte = *spte;
 941                 BUG_ON(!(_spte & PT_PRESENT_MASK));
 942                 _young = _spte & PT_ACCESSED_MASK;
 943                 if (_young) {
 944                         young = 1;
 945                         clear_bit(PT_ACCESSED_SHIFT, (unsigned long *)spte);
 946                 }
 947                 spte = rmap_next(kvm, rmapp, spte);
 948         }
 949         return young;
 950 }
 951
 952 #define RMAP_RECYCLE_THRESHOLD 1000
 953
 954 static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
 955 {
 956         unsigned long *rmapp;
 957         struct kvm_mmu_page *sp;
 958
 959         sp = page_header(__pa(spte));
 960
 961         rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
 962
 963         kvm_unmap_rmapp(vcpu->kvm, rmapp, 0);
 964         kvm_flush_remote_tlbs(vcpu->kvm);
 965 }
 966
 967 int kvm_age_hva(struct kvm *kvm, unsigned long hva)
 968 {
 969         return kvm_handle_hva(kvm, hva, 0, kvm_age_rmapp);
 970 }
 971
 972 #ifdef MMU_DEBUG
 973 static int is_empty_shadow_page(u64 *spt)
 974 {
 975         u64 *pos;
 976         u64 *end;
 977
 978         for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
 979                 if (is_shadow_present_pte(*pos)) {
 980                         printk(KERN_ERR "%s: %p %llx\n", __func__,
 981                                pos, *pos);
 982                         return 0;
 983                 }
 984         return 1;
 985 }
 986 #endif
 987
 988 /*
 989  * This value is the sum of all of the kvm instances's
 990  * kvm->arch.n_used_mmu_pages values.  We need a global,
 991  * aggregate version in order to make the slab shrinker
 992  * faster
 993  */
 994 static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, int nr)
 995 {
 996         kvm->arch.n_used_mmu_pages += nr;
 997         percpu_counter_add(&kvm_total_used_mmu_pages, nr);
 998 }
 999
1000 static void kvm_mmu_free_page(struct kvm *kvm, struct kvm_mmu_page *sp)
1001 {
1002         ASSERT(is_empty_shadow_page(sp->spt));
1003         hlist_del(&sp->hash_link);
1004         list_del(&sp->link);
1005         __free_page(virt_to_page(sp->spt));
1006         if (!sp->role.direct)
1007                 __free_page(virt_to_page(sp->gfns));
1008         kmem_cache_free(mmu_page_header_cache, sp);
1009         kvm_mod_used_mmu_pages(kvm, -1);
1010 }
1011
1012 static unsigned kvm_page_table_hashfn(gfn_t gfn)
1013 {
1014         return gfn & ((1 << KVM_MMU_HASH_SHIFT) - 1);
1015 }
1016
1017 static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
1018                                                u64 *parent_pte, int direct)
1019 {
1020         struct kvm_mmu_page *sp;
1021
1022         sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache, sizeof *sp);
1023         sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE);
1024         if (!direct)
1025                 sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache,
1026                                                   PAGE_SIZE);
1027         set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
1028         list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
1029         bitmap_zero(sp->slot_bitmap, KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS);
1030         sp->multimapped = 0;
1031         sp->parent_pte = parent_pte;
1032         kvm_mod_used_mmu_pages(vcpu->kvm, +1);
1033         return sp;
1034 }
1035
1036 static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
1037                                     struct kvm_mmu_page *sp, u64 *parent_pte)
1038 {
1039         struct kvm_pte_chain *pte_chain;
1040         struct hlist_node *node;
1041         int i;
1042
1043         if (!parent_pte)
1044                 return;
1045         if (!sp->multimapped) {
1046                 u64 *old = sp->parent_pte;
1047
1048                 if (!old) {
1049                         sp->parent_pte = parent_pte;
1050                         return;
1051                 }
1052                 sp->multimapped = 1;
1053                 pte_chain = mmu_alloc_pte_chain(vcpu);
1054                 INIT_HLIST_HEAD(&sp->parent_ptes);
1055                 hlist_add_head(&pte_chain->link, &sp->parent_ptes);
1056                 pte_chain->parent_ptes[0] = old;
1057         }
1058         hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link) {
1059                 if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1])
1060                         continue;
1061                 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i)
1062                         if (!pte_chain->parent_ptes[i]) {
1063                                 pte_chain->parent_ptes[i] = parent_pte;
1064                                 return;
1065                         }
1066         }
1067         pte_chain = mmu_alloc_pte_chain(vcpu);
1068         BUG_ON(!pte_chain);
1069         hlist_add_head(&pte_chain->link, &sp->parent_ptes);
1070         pte_chain->parent_ptes[0] = parent_pte;
1071 }
1072
1073 static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
1074                                        u64 *parent_pte)
1075 {
1076         struct kvm_pte_chain *pte_chain;
1077         struct hlist_node *node;
1078         int i;
1079
1080         if (!sp->multimapped) {
1081                 BUG_ON(sp->parent_pte != parent_pte);
1082                 sp->parent_pte = NULL;
1083                 return;
1084         }
1085         hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
1086                 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
1087                         if (!pte_chain->parent_ptes[i])
1088                                 break;
1089                         if (pte_chain->parent_ptes[i] != parent_pte)
1090                                 continue;
1091                         while (i + 1 < NR_PTE_CHAIN_ENTRIES
1092                                 && pte_chain->parent_ptes[i + 1]) {
1093                                 pte_chain->parent_ptes[i]
1094                                         = pte_chain->parent_ptes[i + 1];
1095                                 ++i;
1096                         }
1097                         pte_chain->parent_ptes[i] = NULL;
1098                         if (i == 0) {
1099                                 hlist_del(&pte_chain->link);
1100                                 mmu_free_pte_chain(pte_chain);
1101                                 if (hlist_empty(&sp->parent_ptes)) {
1102                                         sp->multimapped = 0;
1103                                         sp->parent_pte = NULL;
1104                                 }
1105                         }
1106                         return;
1107                 }
1108         BUG();
1109 }
1110
1111 static void mmu_parent_walk(struct kvm_mmu_page *sp, mmu_parent_walk_fn fn)
1112 {
1113         struct kvm_pte_chain *pte_chain;
1114         struct hlist_node *node;
1115         struct kvm_mmu_page *parent_sp;
1116         int i;
1117
1118         if (!sp->multimapped && sp->parent_pte) {
1119                 parent_sp = page_header(__pa(sp->parent_pte));
1120                 fn(parent_sp, sp->parent_pte);
1121                 return;
1122         }
1123
1124         hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
1125                 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
1126                         u64 *spte = pte_chain->parent_ptes[i];
1127
1128                         if (!spte)
1129                                 break;
1130                         parent_sp = page_header(__pa(spte));
1131                         fn(parent_sp, spte);
1132                 }
1133 }
1134
1135 static void mark_unsync(struct kvm_mmu_page *sp, u64 *spte);
1136 static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)
1137 {
1138         mmu_parent_walk(sp, mark_unsync);
1139 }
1140
1141 static void mark_unsync(struct kvm_mmu_page *sp, u64 *spte)
1142 {
1143         unsigned int index;
1144
1145         index = spte - sp->spt;
1146         if (__test_and_set_bit(index, sp->unsync_child_bitmap))
1147                 return;
1148         if (sp->unsync_children++)
1149                 return;
1150         kvm_mmu_mark_parents_unsync(sp);
1151 }
1152
1153 static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu,
1154                                     struct kvm_mmu_page *sp)
1155 {
1156         int i;
1157
1158         for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
1159                 sp->spt[i] = shadow_trap_nonpresent_pte;
1160 }
1161
1162 static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
1163                                struct kvm_mmu_page *sp, bool clear_unsync)
1164 {
1165         return 1;
1166 }
1167
1168 static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
1169 {
1170 }
1171
1172 #define KVM_PAGE_ARRAY_NR 16
1173
1174 struct kvm_mmu_pages {
1175         struct mmu_page_and_offset {
1176                 struct kvm_mmu_page *sp;
1177                 unsigned int idx;
1178         } page[KVM_PAGE_ARRAY_NR];
1179         unsigned int nr;
1180 };
1181
1182 #define for_each_unsync_children(bitmap, idx)           \
1183         for (idx = find_first_bit(bitmap, 512);         \
1184              idx < 512;                                 \
1185              idx = find_next_bit(bitmap, 512, idx+1))
1186
1187 static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
1188                          int idx)
1189 {
1190         int i;
1191
1192         if (sp->unsync)
1193                 for (i=0; i < pvec->nr; i++)
1194                         if (pvec->page[i].sp == sp)
1195                                 return 0;
1196
1197         pvec->page[pvec->nr].sp = sp;
1198         pvec->page[pvec->nr].idx = idx;
1199         pvec->nr++;
1200         return (pvec->nr == KVM_PAGE_ARRAY_NR);
1201 }
1202
1203 static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
1204                            struct kvm_mmu_pages *pvec)
1205 {
1206         int i, ret, nr_unsync_leaf = 0;
1207
1208         for_each_unsync_children(sp->unsync_child_bitmap, i) {
1209                 struct kvm_mmu_page *child;
1210                 u64 ent = sp->spt[i];
1211
1212                 if (!is_shadow_present_pte(ent) || is_large_pte(ent))
1213                         goto clear_child_bitmap;
1214
1215                 child = page_header(ent & PT64_BASE_ADDR_MASK);
1216
1217                 if (child->unsync_children) {
1218                         if (mmu_pages_add(pvec, child, i))
1219                                 return -ENOSPC;
1220
1221                         ret = __mmu_unsync_walk(child, pvec);
1222                         if (!ret)
1223                                 goto clear_child_bitmap;
1224                         else if (ret > 0)
1225                                 nr_unsync_leaf += ret;
1226                         else
1227                                 return ret;
1228                 } else if (child->unsync) {
1229                         nr_unsync_leaf++;
1230                         if (mmu_pages_add(pvec, child, i))
1231                                 return -ENOSPC;
1232                 } else
1233                          goto clear_child_bitmap;
1234
1235                 continue;
1236
1237 clear_child_bitmap:
1238                 __clear_bit(i, sp->unsync_child_bitmap);
1239                 sp->unsync_children--;
1240                 WARN_ON((int)sp->unsync_children < 0);
1241         }
1242
1243
1244         return nr_unsync_leaf;
1245 }
1246
1247 static int mmu_unsync_walk(struct kvm_mmu_page *sp,
1248                            struct kvm_mmu_pages *pvec)
1249 {
1250         if (!sp->unsync_children)
1251                 return 0;
1252
1253         mmu_pages_add(pvec, sp, 0);
1254         return __mmu_unsync_walk(sp, pvec);
1255 }
1256
1257 static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
1258 {
1259         WARN_ON(!sp->unsync);
1260         trace_kvm_mmu_sync_page(sp);
1261         sp->unsync = 0;
1262         --kvm->stat.mmu_unsync;
1263 }
1264
1265 static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
1266                                     struct list_head *invalid_list);
1267 static void kvm_mmu_commit_zap_page(struct kvm *kvm,
1268                                     struct list_head *invalid_list);
1269
1270 #define for_each_gfn_sp(kvm, sp, gfn, pos)                              \
1271   hlist_for_each_entry(sp, pos,                                         \
1272    &(kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)], hash_link)   \
1273         if ((sp)->gfn != (gfn)) {} else
1274
1275 #define for_each_gfn_indirect_valid_sp(kvm, sp, gfn, pos)               \
1276   hlist_for_each_entry(sp, pos,                                         \
1277    &(kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)], hash_link)   \
1278                 if ((sp)->gfn != (gfn) || (sp)->role.direct ||          \
1279                         (sp)->role.invalid) {} else
1280
1281 /* @sp->gfn should be write-protected at the call site */
1282 static int __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
1283                            struct list_head *invalid_list, bool clear_unsync)
1284 {
1285         if (sp->role.cr4_pae != !!is_pae(vcpu)) {
1286                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
1287                 return 1;
1288         }
1289
1290         if (clear_unsync)
1291                 kvm_unlink_unsync_page(vcpu->kvm, sp);
1292
1293         if (vcpu->arch.mmu.sync_page(vcpu, sp, clear_unsync)) {
1294                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
1295                 return 1;
1296         }
1297
1298         kvm_mmu_flush_tlb(vcpu);
1299         return 0;
1300 }
1301
1302 static int kvm_sync_page_transient(struct kvm_vcpu *vcpu,
1303                                    struct kvm_mmu_page *sp)
1304 {
1305         LIST_HEAD(invalid_list);
1306         int ret;
1307
1308         ret = __kvm_sync_page(vcpu, sp, &invalid_list, false);
1309         if (ret)
1310                 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
1311
1312         return ret;
1313 }
1314
1315 static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
1316                          struct list_head *invalid_list)
1317 {
1318         return __kvm_sync_page(vcpu, sp, invalid_list, true);
1319 }
1320
1321 /* @gfn should be write-protected at the call site */
1322 static void kvm_sync_pages(struct kvm_vcpu *vcpu,  gfn_t gfn)
1323 {
1324         struct kvm_mmu_page *s;
1325         struct hlist_node *node;
1326         LIST_HEAD(invalid_list);
1327         bool flush = false;
1328
1329         for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn, node) {
1330                 if (!s->unsync)
1331                         continue;
1332
1333                 WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
1334                 if ((s->role.cr4_pae != !!is_pae(vcpu)) ||
1335                         (vcpu->arch.mmu.sync_page(vcpu, s, true))) {
1336                         kvm_mmu_prepare_zap_page(vcpu->kvm, s, &invalid_list);
1337                         continue;
1338                 }
1339                 kvm_unlink_unsync_page(vcpu->kvm, s);
1340                 flush = true;
1341         }
1342
1343         kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
1344         if (flush)
1345                 kvm_mmu_flush_tlb(vcpu);
1346 }
1347
1348 struct mmu_page_path {
1349         struct kvm_mmu_page *parent[PT64_ROOT_LEVEL-1];
1350         unsigned int idx[PT64_ROOT_LEVEL-1];
1351 };
1352
1353 #define for_each_sp(pvec, sp, parents, i)                       \
1354                 for (i = mmu_pages_next(&pvec, &parents, -1),   \
1355                         sp = pvec.page[i].sp;                   \
1356                         i < pvec.nr && ({ sp = pvec.page[i].sp; 1;});   \
1357                         i = mmu_pages_next(&pvec, &parents, i))
1358
1359 static int mmu_pages_next(struct kvm_mmu_pages *pvec,
1360                           struct mmu_page_path *parents,
1361                           int i)
1362 {
1363         int n;
1364
1365         for (n = i+1; n < pvec->nr; n++) {
1366                 struct kvm_mmu_page *sp = pvec->page[n].sp;
1367
1368                 if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
1369                         parents->idx[0] = pvec->page[n].idx;
1370                         return n;
1371                 }
1372
1373                 parents->parent[sp->role.level-2] = sp;
1374                 parents->idx[sp->role.level-1] = pvec->page[n].idx;
1375         }
1376
1377         return n;
1378 }
1379
1380 static void mmu_pages_clear_parents(struct mmu_page_path *parents)
1381 {
1382         struct kvm_mmu_page *sp;
1383         unsigned int level = 0;
1384
1385         do {
1386                 unsigned int idx = parents->idx[level];
1387
1388                 sp = parents->parent[level];
1389                 if (!sp)
1390                         return;
1391
1392                 --sp->unsync_children;
1393                 WARN_ON((int)sp->unsync_children < 0);
1394                 __clear_bit(idx, sp->unsync_child_bitmap);
1395                 level++;
1396         } while (level < PT64_ROOT_LEVEL-1 && !sp->unsync_children);
1397 }
1398
1399 static void kvm_mmu_pages_init(struct kvm_mmu_page *parent,
1400                                struct mmu_page_path *parents,
1401                                struct kvm_mmu_pages *pvec)
1402 {
1403         parents->parent[parent->role.level-1] = NULL;
1404         pvec->nr = 0;
1405 }
1406
1407 static void mmu_sync_children(struct kvm_vcpu *vcpu,
1408                               struct kvm_mmu_page *parent)
1409 {
1410         int i;
1411         struct kvm_mmu_page *sp;
1412         struct mmu_page_path parents;
1413         struct kvm_mmu_pages pages;
1414         LIST_HEAD(invalid_list);
1415
1416         kvm_mmu_pages_init(parent, &parents, &pages);
1417         while (mmu_unsync_walk(parent, &pages)) {
1418                 int protected = 0;
1419
1420                 for_each_sp(pages, sp, parents, i)
1421                         protected |= rmap_write_protect(vcpu->kvm, sp->gfn);
1422
1423                 if (protected)
1424                         kvm_flush_remote_tlbs(vcpu->kvm);
1425
1426                 for_each_sp(pages, sp, parents, i) {
1427                         kvm_sync_page(vcpu, sp, &invalid_list);
1428                         mmu_pages_clear_parents(&parents);
1429                 }
1430                 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
1431                 cond_resched_lock(&vcpu->kvm->mmu_lock);
1432                 kvm_mmu_pages_init(parent, &parents, &pages);
1433         }
1434 }
1435
1436 static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
1437                                              gfn_t gfn,
1438                                              gva_t gaddr,
1439                                              unsigned level,
1440                                              int direct,
1441                                              unsigned access,
1442                                              u64 *parent_pte)
1443 {
1444         union kvm_mmu_page_role role;
1445         unsigned quadrant;
1446         struct kvm_mmu_page *sp;
1447         struct hlist_node *node;
1448         bool need_sync = false;
1449
1450         role = vcpu->arch.mmu.base_role;
1451         role.level = level;
1452         role.direct = direct;
1453         if (role.direct)
1454                 role.cr4_pae = 0;
1455         role.access = access;
1456         if (!vcpu->arch.mmu.direct_map
1457             && vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) {
1458                 quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
1459                 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
1460                 role.quadrant = quadrant;
1461         }
1462         for_each_gfn_sp(vcpu->kvm, sp, gfn, node) {
1463                 if (!need_sync && sp->unsync)
1464                         need_sync = true;
1465
1466                 if (sp->role.word != role.word)
1467                         continue;
1468
1469                 if (sp->unsync && kvm_sync_page_transient(vcpu, sp))
1470                         break;
1471
1472                 mmu_page_add_parent_pte(vcpu, sp, parent_pte);
1473                 if (sp->unsync_children) {
1474                         kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
1475                         kvm_mmu_mark_parents_unsync(sp);
1476                 } else if (sp->unsync)
1477                         kvm_mmu_mark_parents_unsync(sp);
1478
1479                 trace_kvm_mmu_get_page(sp, false);
1480                 return sp;
1481         }
1482         ++vcpu->kvm->stat.mmu_cache_miss;
1483         sp = kvm_mmu_alloc_page(vcpu, parent_pte, direct);
1484         if (!sp)
1485                 return sp;
1486         sp->gfn = gfn;
1487         sp->role = role;
1488         hlist_add_head(&sp->hash_link,
1489                 &vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)]);
1490         if (!direct) {
1491                 if (rmap_write_protect(vcpu->kvm, gfn))
1492                         kvm_flush_remote_tlbs(vcpu->kvm);
1493                 if (level > PT_PAGE_TABLE_LEVEL && need_sync)
1494                         kvm_sync_pages(vcpu, gfn);
1495
1496                 account_shadowed(vcpu->kvm, gfn);
1497         }
1498         if (shadow_trap_nonpresent_pte != shadow_notrap_nonpresent_pte)
1499                 vcpu->arch.mmu.prefetch_page(vcpu, sp);
1500         else
1501                 nonpaging_prefetch_page(vcpu, sp);
1502         trace_kvm_mmu_get_page(sp, true);
1503         return sp;
1504 }
1505
1506 static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
1507                              struct kvm_vcpu *vcpu, u64 addr)
1508 {
1509         iterator->addr = addr;
1510         iterator->shadow_addr = vcpu->arch.mmu.root_hpa;
1511         iterator->level = vcpu->arch.mmu.shadow_root_level;
1512
1513         if (iterator->level == PT64_ROOT_LEVEL &&
1514             vcpu->arch.mmu.root_level < PT64_ROOT_LEVEL &&
1515             !vcpu->arch.mmu.direct_map)
1516                 --iterator->level;
1517
1518         if (iterator->level == PT32E_ROOT_LEVEL) {
1519                 iterator->shadow_addr
1520                         = vcpu->arch.mmu.pae_root[(addr >> 30) & 3];
1521                 iterator->shadow_addr &= PT64_BASE_ADDR_MASK;
1522                 --iterator->level;
1523                 if (!iterator->shadow_addr)
1524                         iterator->level = 0;
1525         }
1526 }
1527
1528 static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
1529 {
1530         if (iterator->level < PT_PAGE_TABLE_LEVEL)
1531                 return false;
1532
1533         if (iterator->level == PT_PAGE_TABLE_LEVEL)
1534                 if (is_large_pte(*iterator->sptep))
1535                         return false;
1536
1537         iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
1538         iterator->sptep = ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
1539         return true;
1540 }
1541
1542 static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
1543 {
1544         iterator->shadow_addr = *iterator->sptep & PT64_BASE_ADDR_MASK;
1545         --iterator->level;
1546 }
1547
1548 static void link_shadow_page(u64 *sptep, struct kvm_mmu_page *sp)
1549 {
1550         u64 spte;
1551
1552         spte = __pa(sp->spt)
1553                 | PT_PRESENT_MASK | PT_ACCESSED_MASK
1554                 | PT_WRITABLE_MASK | PT_USER_MASK;
1555         __set_spte(sptep, spte);
1556 }
1557
1558 static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
1559 {
1560         if (is_large_pte(*sptep)) {
1561                 drop_spte(vcpu->kvm, sptep, shadow_trap_nonpresent_pte);
1562                 kvm_flush_remote_tlbs(vcpu->kvm);
1563         }
1564 }
1565
1566 static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
1567                                    unsigned direct_access)
1568 {
1569         if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {
1570                 struct kvm_mmu_page *child;
1571
1572                 /*
1573                  * For the direct sp, if the guest pte's dirty bit
1574                  * changed form clean to dirty, it will corrupt the
1575                  * sp's access: allow writable in the read-only sp,
1576                  * so we should update the spte at this point to get
1577                  * a new sp with the correct access.
1578                  */
1579                 child = page_header(*sptep & PT64_BASE_ADDR_MASK);
1580                 if (child->role.access == direct_access)
1581                         return;
1582
1583                 mmu_page_remove_parent_pte(child, sptep);
1584                 __set_spte(sptep, shadow_trap_nonpresent_pte);
1585                 kvm_flush_remote_tlbs(vcpu->kvm);
1586         }
1587 }
1588
1589 static void kvm_mmu_page_unlink_children(struct kvm *kvm,
1590                                          struct kvm_mmu_page *sp)
1591 {
1592         unsigned i;
1593         u64 *pt;
1594         u64 ent;
1595
1596         pt = sp->spt;
1597
1598         for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
1599                 ent = pt[i];
1600
1601                 if (is_shadow_present_pte(ent)) {
1602                         if (!is_last_spte(ent, sp->role.level)) {
1603                                 ent &= PT64_BASE_ADDR_MASK;
1604                                 mmu_page_remove_parent_pte(page_header(ent),
1605                                                            &pt[i]);
1606                         } else {
1607                                 if (is_large_pte(ent))
1608                                         --kvm->stat.lpages;
1609                                 drop_spte(kvm, &pt[i],
1610                                           shadow_trap_nonpresent_pte);
1611                         }
1612                 }
1613                 pt[i] = shadow_trap_nonpresent_pte;
1614         }
1615 }
1616
1617 static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte)
1618 {
1619         mmu_page_remove_parent_pte(sp, parent_pte);
1620 }
1621
1622 static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm)
1623 {
1624         int i;
1625         struct kvm_vcpu *vcpu;
1626
1627         kvm_for_each_vcpu(i, vcpu, kvm)
1628                 vcpu->arch.last_pte_updated = NULL;
1629 }
1630
1631 static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
1632 {
1633         u64 *parent_pte;
1634
1635         while (sp->multimapped || sp->parent_pte) {
1636                 if (!sp->multimapped)
1637                         parent_pte = sp->parent_pte;
1638                 else {
1639                         struct kvm_pte_chain *chain;
1640
1641                         chain = container_of(sp->parent_ptes.first,
1642                                              struct kvm_pte_chain, link);
1643                         parent_pte = chain->parent_ptes[0];
1644                 }
1645                 BUG_ON(!parent_pte);
1646                 kvm_mmu_put_page(sp, parent_pte);
1647                 __set_spte(parent_pte, shadow_trap_nonpresent_pte);
1648         }
1649 }
1650
1651 static int mmu_zap_unsync_children(struct kvm *kvm,
1652                                    struct kvm_mmu_page *parent,
1653                                    struct list_head *invalid_list)
1654 {
1655         int i, zapped = 0;
1656         struct mmu_page_path parents;
1657         struct kvm_mmu_pages pages;
1658
1659         if (parent->role.level == PT_PAGE_TABLE_LEVEL)
1660                 return 0;
1661
1662         kvm_mmu_pages_init(parent, &parents, &pages);
1663         while (mmu_unsync_walk(parent, &pages)) {
1664                 struct kvm_mmu_page *sp;
1665
1666                 for_each_sp(pages, sp, parents, i) {
1667                         kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
1668                         mmu_pages_clear_parents(&parents);
1669                         zapped++;
1670                 }
1671                 kvm_mmu_pages_init(parent, &parents, &pages);
1672         }
1673
1674         return zapped;
1675 }
1676
1677 static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
1678                                     struct list_head *invalid_list)
1679 {
1680         int ret;
1681
1682         trace_kvm_mmu_prepare_zap_page(sp);
1683         ++kvm->stat.mmu_shadow_zapped;
1684         ret = mmu_zap_unsync_children(kvm, sp, invalid_list);
1685         kvm_mmu_page_unlink_children(kvm, sp);
1686         kvm_mmu_unlink_parents(kvm, sp);
1687         if (!sp->role.invalid && !sp->role.direct)
1688                 unaccount_shadowed(kvm, sp->gfn);
1689         if (sp->unsync)
1690                 kvm_unlink_unsync_page(kvm, sp);
1691         if (!sp->root_count) {
1692                 /* Count self */
1693                 ret++;
1694                 list_move(&sp->link, invalid_list);
1695         } else {
1696                 list_move(&sp->link, &kvm->arch.active_mmu_pages);
1697                 kvm_reload_remote_mmus(kvm);
1698         }
1699
1700         sp->role.invalid = 1;
1701         kvm_mmu_reset_last_pte_updated(kvm);
1702         return ret;
1703 }
1704
1705 static void kvm_mmu_commit_zap_page(struct kvm *kvm,
1706                                     struct list_head *invalid_list)
1707 {
1708         struct kvm_mmu_page *sp;
1709
1710         if (list_empty(invalid_list))
1711                 return;
1712
1713         kvm_flush_remote_tlbs(kvm);
1714
1715         do {
1716                 sp = list_first_entry(invalid_list, struct kvm_mmu_page, link);
1717                 WARN_ON(!sp->role.invalid || sp->root_count);
1718                 kvm_mmu_free_page(kvm, sp);
1719         } while (!list_empty(invalid_list));
1720
1721 }
1722
1723 /*
1724  * Changing the number of mmu pages allocated to the vm
1725  * Note: if goal_nr_mmu_pages is too small, you will get dead lock
1726  */
1727 void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int goal_nr_mmu_pages)
1728 {
1729         LIST_HEAD(invalid_list);
1730         /*
1731          * If we set the number of mmu pages to be smaller be than the
1732          * number of actived pages , we must to free some mmu pages before we
1733          * change the value
1734          */
1735
1736         if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
1737                 while (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages &&
1738                         !list_empty(&kvm->arch.active_mmu_pages)) {
1739                         struct kvm_mmu_page *page;
1740
1741                         page = container_of(kvm->arch.active_mmu_pages.prev,
1742                                             struct kvm_mmu_page, link);
1743                         kvm_mmu_prepare_zap_page(kvm, page, &invalid_list);
1744                         kvm_mmu_commit_zap_page(kvm, &invalid_list);
1745                 }
1746                 goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages;
1747         }
1748
1749         kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
1750 }
1751
1752 static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
1753 {
1754         struct kvm_mmu_page *sp;
1755         struct hlist_node *node;
1756         LIST_HEAD(invalid_list);
1757         int r;
1758
1759         pgprintk("%s: looking for gfn %llx\n", __func__, gfn);
1760         r = 0;
1761
1762         for_each_gfn_indirect_valid_sp(kvm, sp, gfn, node) {
1763                 pgprintk("%s: gfn %llx role %x\n", __func__, gfn,
1764                          sp->role.word);
1765                 r = 1;
1766                 kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
1767         }
1768         kvm_mmu_commit_zap_page(kvm, &invalid_list);
1769         return r;
1770 }
1771
1772 static void mmu_unshadow(struct kvm *kvm, gfn_t gfn)
1773 {
1774         struct kvm_mmu_page *sp;
1775         struct hlist_node *node;
1776         LIST_HEAD(invalid_list);
1777
1778         for_each_gfn_indirect_valid_sp(kvm, sp, gfn, node) {
1779                 pgprintk("%s: zap %llx %x\n",
1780                          __func__, gfn, sp->role.word);
1781                 kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
1782         }
1783         kvm_mmu_commit_zap_page(kvm, &invalid_list);
1784 }
1785
1786 static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn)
1787 {
1788         int slot = memslot_id(kvm, gfn);
1789         struct kvm_mmu_page *sp = page_header(__pa(pte));
1790
1791         __set_bit(slot, sp->slot_bitmap);
1792 }
1793
1794 static void mmu_convert_notrap(struct kvm_mmu_page *sp)
1795 {
1796         int i;
1797         u64 *pt = sp->spt;
1798
1799         if (shadow_trap_nonpresent_pte == shadow_notrap_nonpresent_pte)
1800                 return;
1801
1802         for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
1803                 if (pt[i] == shadow_notrap_nonpresent_pte)
1804                         __set_spte(&pt[i], shadow_trap_nonpresent_pte);
1805         }
1806 }
1807
1808 /*
1809  * The function is based on mtrr_type_lookup() in
1810  * arch/x86/kernel/cpu/mtrr/generic.c
1811  */
1812 static int get_mtrr_type(struct mtrr_state_type *mtrr_state,
1813                          u64 start, u64 end)
1814 {
1815         int i;
1816         u64 base, mask;
1817         u8 prev_match, curr_match;
1818         int num_var_ranges = KVM_NR_VAR_MTRR;
1819
1820         if (!mtrr_state->enabled)
1821                 return 0xFF;
1822
1823         /* Make end inclusive end, instead of exclusive */
1824         end--;
1825
1826         /* Look in fixed ranges. Just return the type as per start */
1827         if (mtrr_state->have_fixed && (start < 0x100000)) {
1828                 int idx;
1829
1830                 if (start < 0x80000) {
1831                         idx = 0;
1832                         idx += (start >> 16);
1833                         return mtrr_state->fixed_ranges[idx];
1834                 } else if (start < 0xC0000) {
1835                         idx = 1 * 8;
1836                         idx += ((start - 0x80000) >> 14);
1837                         return mtrr_state->fixed_ranges[idx];
1838                 } else if (start < 0x1000000) {
1839                         idx = 3 * 8;
1840                         idx += ((start - 0xC0000) >> 12);
1841                         return mtrr_state->fixed_ranges[idx];
1842                 }
1843         }
1844
1845         /*
1846          * Look in variable ranges
1847          * Look of multiple ranges matching this address and pick type
1848          * as per MTRR precedence
1849          */
1850         if (!(mtrr_state->enabled & 2))
1851                 return mtrr_state->def_type;
1852
1853         prev_match = 0xFF;
1854         for (i = 0; i < num_var_ranges; ++i) {
1855                 unsigned short start_state, end_state;
1856
1857                 if (!(mtrr_state->var_ranges[i].mask_lo & (1 << 11)))
1858                         continue;
1859
1860                 base = (((u64)mtrr_state->var_ranges[i].base_hi) << 32) +
1861                        (mtrr_state->var_ranges[i].base_lo & PAGE_MASK);
1862                 mask = (((u64)mtrr_state->var_ranges[i].mask_hi) << 32) +
1863                        (mtrr_state->var_ranges[i].mask_lo & PAGE_MASK);
1864
1865                 start_state = ((start & mask) == (base & mask));
1866                 end_state = ((end & mask) == (base & mask));
1867                 if (start_state != end_state)
1868                         return 0xFE;
1869
1870                 if ((start & mask) != (base & mask))
1871                         continue;
1872
1873                 curr_match = mtrr_state->var_ranges[i].base_lo & 0xff;
1874                 if (prev_match == 0xFF) {
1875                         prev_match = curr_match;
1876                         continue;
1877                 }
1878
1879                 if (prev_match == MTRR_TYPE_UNCACHABLE ||
1880                     curr_match == MTRR_TYPE_UNCACHABLE)
1881                         return MTRR_TYPE_UNCACHABLE;
1882
1883                 if ((prev_match == MTRR_TYPE_WRBACK &&
1884                      curr_match == MTRR_TYPE_WRTHROUGH) ||
1885                     (prev_match == MTRR_TYPE_WRTHROUGH &&
1886                      curr_match == MTRR_TYPE_WRBACK)) {
1887                         prev_match = MTRR_TYPE_WRTHROUGH;
1888                         curr_match = MTRR_TYPE_WRTHROUGH;
1889                 }
1890
1891                 if (prev_match != curr_match)
1892                         return MTRR_TYPE_UNCACHABLE;
1893         }
1894
1895         if (prev_match != 0xFF)
1896                 return prev_match;
1897
1898         return mtrr_state->def_type;
1899 }
1900
1901 u8 kvm_get_guest_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn)
1902 {
1903         u8 mtrr;
1904
1905         mtrr = get_mtrr_type(&vcpu->arch.mtrr_state, gfn << PAGE_SHIFT,
1906                              (gfn << PAGE_SHIFT) + PAGE_SIZE);
1907         if (mtrr == 0xfe || mtrr == 0xff)
1908                 mtrr = MTRR_TYPE_WRBACK;
1909         return mtrr;
1910 }
1911 EXPORT_SYMBOL_GPL(kvm_get_guest_memory_type);
1912
1913 static void __kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
1914 {
1915         trace_kvm_mmu_unsync_page(sp);
1916         ++vcpu->kvm->stat.mmu_unsync;
1917         sp->unsync = 1;
1918
1919         kvm_mmu_mark_parents_unsync(sp);
1920         mmu_convert_notrap(sp);
1921 }
1922
1923 static void kvm_unsync_pages(struct kvm_vcpu *vcpu,  gfn_t gfn)
1924 {
1925         struct kvm_mmu_page *s;
1926         struct hlist_node *node;
1927
1928         for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn, node) {
1929                 if (s->unsync)
1930                         continue;
1931                 WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL);
1932                 __kvm_unsync_page(vcpu, s);
1933         }
1934 }
1935
1936 static int mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
1937                                   bool can_unsync)
1938 {
1939         struct kvm_mmu_page *s;
1940         struct hlist_node *node;
1941         bool need_unsync = false;
1942
1943         for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn, node) {
1944                 if (!can_unsync)
1945                         return 1;
1946
1947                 if (s->role.level != PT_PAGE_TABLE_LEVEL)
1948                         return 1;
1949
1950                 if (!need_unsync && !s->unsync) {
1951                         if (!oos_shadow)
1952                                 return 1;
1953                         need_unsync = true;
1954                 }
1955         }
1956         if (need_unsync)
1957                 kvm_unsync_pages(vcpu, gfn);
1958         return 0;
1959 }
1960
1961 static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
1962                     unsigned pte_access, int user_fault,
1963                     int write_fault, int dirty, int level,
1964                     gfn_t gfn, pfn_t pfn, bool speculative,
1965                     bool can_unsync, bool reset_host_protection)
1966 {
1967         u64 spte;
1968         int ret = 0;
1969
1970         /*
1971          * We don't set the accessed bit, since we sometimes want to see
1972          * whether the guest actually used the pte (in order to detect
1973          * demand paging).
1974          */
1975         spte = shadow_base_present_pte;
1976         if (!speculative)
1977                 spte |= shadow_accessed_mask;
1978         if (!dirty)
1979                 pte_access &= ~ACC_WRITE_MASK;
1980         if (pte_access & ACC_EXEC_MASK)
1981                 spte |= shadow_x_mask;
1982         else
1983                 spte |= shadow_nx_mask;
1984         if (pte_access & ACC_USER_MASK)
1985                 spte |= shadow_user_mask;
1986         if (level > PT_PAGE_TABLE_LEVEL)
1987                 spte |= PT_PAGE_SIZE_MASK;
1988         if (tdp_enabled)
1989                 spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn,
1990                         kvm_is_mmio_pfn(pfn));
1991
1992         if (reset_host_protection)
1993                 spte |= SPTE_HOST_WRITEABLE;
1994
1995         spte |= (u64)pfn << PAGE_SHIFT;
1996
1997         if ((pte_access & ACC_WRITE_MASK)
1998             || (!vcpu->arch.mmu.direct_map && write_fault
1999                 && !is_write_protection(vcpu) && !user_fault)) {
2000
2001                 if (level > PT_PAGE_TABLE_LEVEL &&
2002                     has_wrprotected_page(vcpu->kvm, gfn, level)) {
2003                         ret = 1;
2004                         drop_spte(vcpu->kvm, sptep, shadow_trap_nonpresent_pte);
2005                         goto done;
2006                 }
2007
2008                 spte |= PT_WRITABLE_MASK;
2009
2010                 if (!vcpu->arch.mmu.direct_map
2011                     && !(pte_access & ACC_WRITE_MASK))
2012                         spte &= ~PT_USER_MASK;
2013
2014                 /*
2015                  * Optimization: for pte sync, if spte was writable the hash
2016                  * lookup is unnecessary (and expensive). Write protection
2017                  * is responsibility of mmu_get_page / kvm_sync_page.
2018                  * Same reasoning can be applied to dirty page accounting.
2019                  */
2020                 if (!can_unsync && is_writable_pte(*sptep))
2021                         goto set_pte;
2022
2023                 if (mmu_need_write_protect(vcpu, gfn, can_unsync)) {
2024                         pgprintk("%s: found shadow page for %llx, marking ro\n",
2025                                  __func__, gfn);
2026                         ret = 1;
2027                         pte_access &= ~ACC_WRITE_MASK;
2028                         if (is_writable_pte(spte))
2029                                 spte &= ~PT_WRITABLE_MASK;
2030                 }
2031         }
2032
2033         if (pte_access & ACC_WRITE_MASK)
2034                 mark_page_dirty(vcpu->kvm, gfn);
2035
2036 set_pte:
2037         update_spte(sptep, spte);
2038 done:
2039         return ret;
2040 }
2041
2042 static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2043                          unsigned pt_access, unsigned pte_access,
2044                          int user_fault, int write_fault, int dirty,
2045                          int *ptwrite, int level, gfn_t gfn,
2046                          pfn_t pfn, bool speculative,
2047                          bool reset_host_protection)
2048 {
2049         int was_rmapped = 0;
2050         int rmap_count;
2051
2052         pgprintk("%s: spte %llx access %x write_fault %d"
2053                  " user_fault %d gfn %llx\n",
2054                  __func__, *sptep, pt_access,
2055                  write_fault, user_fault, gfn);
2056
2057         if (is_rmap_spte(*sptep)) {
2058                 /*
2059                  * If we overwrite a PTE page pointer with a 2MB PMD, unlink
2060                  * the parent of the now unreachable PTE.
2061                  */
2062                 if (level > PT_PAGE_TABLE_LEVEL &&
2063                     !is_large_pte(*sptep)) {
2064                         struct kvm_mmu_page *child;
2065                         u64 pte = *sptep;
2066
2067                         child = page_header(pte & PT64_BASE_ADDR_MASK);
2068                         mmu_page_remove_parent_pte(child, sptep);
2069                         __set_spte(sptep, shadow_trap_nonpresent_pte);
2070                         kvm_flush_remote_tlbs(vcpu->kvm);
2071                 } else if (pfn != spte_to_pfn(*sptep)) {
2072                         pgprintk("hfn old %llx new %llx\n",
2073                                  spte_to_pfn(*sptep), pfn);
2074                         drop_spte(vcpu->kvm, sptep, shadow_trap_nonpresent_pte);
2075                         kvm_flush_remote_tlbs(vcpu->kvm);
2076                 } else
2077                         was_rmapped = 1;
2078         }
2079
2080         if (set_spte(vcpu, sptep, pte_access, user_fault, write_fault,
2081                       dirty, level, gfn, pfn, speculative, true,
2082                       reset_host_protection)) {
2083                 if (write_fault)
2084                         *ptwrite = 1;
2085                 kvm_mmu_flush_tlb(vcpu);
2086         }
2087
2088         pgprintk("%s: setting spte %llx\n", __func__, *sptep);
2089         pgprintk("instantiating %s PTE (%s) at %llx (%llx) addr %p\n",
2090                  is_large_pte(*sptep)? "2MB" : "4kB",
2091                  *sptep & PT_PRESENT_MASK ?"RW":"R", gfn,
2092                  *sptep, sptep);
2093         if (!was_rmapped && is_large_pte(*sptep))
2094                 ++vcpu->kvm->stat.lpages;
2095
2096         page_header_update_slot(vcpu->kvm, sptep, gfn);
2097         if (!was_rmapped) {
2098                 rmap_count = rmap_add(vcpu, sptep, gfn);
2099                 if (rmap_count > RMAP_RECYCLE_THRESHOLD)
2100                         rmap_recycle(vcpu, sptep, gfn);
2101         }
2102         kvm_release_pfn_clean(pfn);
2103         if (speculative) {
2104                 vcpu->arch.last_pte_updated = sptep;
2105                 vcpu->arch.last_pte_gfn = gfn;
2106         }
2107 }
2108
2109 static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
2110 {
2111 }
2112
2113 static struct kvm_memory_slot *
2114 pte_prefetch_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn, bool no_dirty_log)
2115 {
2116         struct kvm_memory_slot *slot;
2117
2118         slot = gfn_to_memslot(vcpu->kvm, gfn);
2119         if (!slot || slot->flags & KVM_MEMSLOT_INVALID ||
2120               (no_dirty_log && slot->dirty_bitmap))
2121                 slot = NULL;
2122
2123         return slot;
2124 }
2125
2126 static pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn,
2127                                      bool no_dirty_log)
2128 {
2129         struct kvm_memory_slot *slot;
2130         unsigned long hva;
2131
2132         slot = pte_prefetch_gfn_to_memslot(vcpu, gfn, no_dirty_log);
2133         if (!slot) {
2134                 get_page(bad_page);
2135                 return page_to_pfn(bad_page);
2136         }
2137
2138         hva = gfn_to_hva_memslot(slot, gfn);
2139
2140         return hva_to_pfn_atomic(vcpu->kvm, hva);
2141 }
2142
2143 static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
2144                                     struct kvm_mmu_page *sp,
2145                                     u64 *start, u64 *end)
2146 {
2147         struct page *pages[PTE_PREFETCH_NUM];
2148         unsigned access = sp->role.access;
2149         int i, ret;
2150         gfn_t gfn;
2151
2152         gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt);
2153         if (!pte_prefetch_gfn_to_memslot(vcpu, gfn, access & ACC_WRITE_MASK))
2154                 return -1;
2155
2156         ret = gfn_to_page_many_atomic(vcpu->kvm, gfn, pages, end - start);
2157         if (ret <= 0)
2158                 return -1;
2159
2160         for (i = 0; i < ret; i++, gfn++, start++)
2161                 mmu_set_spte(vcpu, start, ACC_ALL,
2162                              access, 0, 0, 1, NULL,
2163                              sp->role.level, gfn,
2164                              page_to_pfn(pages[i]), true, true);
2165
2166         return 0;
2167 }
2168
2169 static void __direct_pte_prefetch(struct kvm_vcpu *vcpu,
2170                                   struct kvm_mmu_page *sp, u64 *sptep)
2171 {
2172         u64 *spte, *start = NULL;
2173         int i;
2174
2175         WARN_ON(!sp->role.direct);
2176
2177         i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
2178         spte = sp->spt + i;
2179
2180         for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
2181                 if (*spte != shadow_trap_nonpresent_pte || spte == sptep) {
2182                         if (!start)
2183                                 continue;
2184                         if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0)
2185                                 break;
2186                         start = NULL;
2187                 } else if (!start)
2188                         start = spte;
2189         }
2190 }
2191
2192 static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
2193 {
2194         struct kvm_mmu_page *sp;
2195
2196         /*
2197          * Since it's no accessed bit on EPT, it's no way to
2198          * distinguish between actually accessed translations
2199          * and prefetched, so disable pte prefetch if EPT is
2200          * enabled.
2201          */
2202         if (!shadow_accessed_mask)
2203                 return;
2204
2205         sp = page_header(__pa(sptep));
2206         if (sp->role.level > PT_PAGE_TABLE_LEVEL)
2207                 return;
2208
2209         __direct_pte_prefetch(vcpu, sp, sptep);
2210 }
2211
2212 static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
2213                         int level, gfn_t gfn, pfn_t pfn)
2214 {
2215         struct kvm_shadow_walk_iterator iterator;
2216         struct kvm_mmu_page *sp;
2217         int pt_write = 0;
2218         gfn_t pseudo_gfn;
2219
2220         for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
2221                 if (iterator.level == level) {
2222                         mmu_set_spte(vcpu, iterator.sptep, ACC_ALL, ACC_ALL,
2223                                      0, write, 1, &pt_write,
2224                                      level, gfn, pfn, false, true);
2225                         direct_pte_prefetch(vcpu, iterator.sptep);
2226                         ++vcpu->stat.pf_fixed;
2227                         break;
2228                 }
2229
2230                 if (*iterator.sptep == shadow_trap_nonpresent_pte) {
2231                         u64 base_addr = iterator.addr;
2232
2233                         base_addr &= PT64_LVL_ADDR_MASK(iterator.level);
2234                         pseudo_gfn = base_addr >> PAGE_SHIFT;
2235                         sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr,
2236                                               iterator.level - 1,
2237                                               1, ACC_ALL, iterator.sptep);
2238                         if (!sp) {
2239                                 pgprintk("nonpaging_map: ENOMEM\n");
2240                                 kvm_release_pfn_clean(pfn);
2241                                 return -ENOMEM;
2242                         }
2243
2244                         __set_spte(iterator.sptep,
2245                                    __pa(sp->spt)
2246                                    | PT_PRESENT_MASK | PT_WRITABLE_MASK
2247                                    | shadow_user_mask | shadow_x_mask
2248                                    | shadow_accessed_mask);
2249                 }
2250         }
2251         return pt_write;
2252 }
2253
2254 static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *tsk)
2255 {
2256         siginfo_t info;
2257
2258         info.si_signo   = SIGBUS;
2259         info.si_errno   = 0;
2260         info.si_code    = BUS_MCEERR_AR;
2261         info.si_addr    = (void __user *)address;
2262         info.si_addr_lsb = PAGE_SHIFT;
2263
2264         send_sig_info(SIGBUS, &info, tsk);
2265 }
2266
2267 static int kvm_handle_bad_page(struct kvm *kvm, gfn_t gfn, pfn_t pfn)
2268 {
2269         kvm_release_pfn_clean(pfn);
2270         if (is_hwpoison_pfn(pfn)) {
2271                 kvm_send_hwpoison_signal(gfn_to_hva(kvm, gfn), current);
2272                 return 0;
2273         } else if (is_fault_pfn(pfn))
2274                 return -EFAULT;
2275
2276         return 1;
2277 }
2278
2279 static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn)
2280 {
2281         int r;
2282         int level;
2283         pfn_t pfn;
2284         unsigned long mmu_seq;
2285
2286         level = mapping_level(vcpu, gfn);
2287
2288         /*
2289          * This path builds a PAE pagetable - so we can map 2mb pages at
2290          * maximum. Therefore check if the level is larger than that.
2291          */
2292         if (level > PT_DIRECTORY_LEVEL)
2293                 level = PT_DIRECTORY_LEVEL;
2294
2295         gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
2296
2297         mmu_seq = vcpu->kvm->mmu_notifier_seq;
2298         smp_rmb();
2299         pfn = gfn_to_pfn(vcpu->kvm, gfn);
2300
2301         /* mmio */
2302         if (is_error_pfn(pfn))
2303                 return kvm_handle_bad_page(vcpu->kvm, gfn, pfn);
2304
2305         spin_lock(&vcpu->kvm->mmu_lock);
2306         if (mmu_notifier_retry(vcpu, mmu_seq))
2307                 goto out_unlock;
2308         kvm_mmu_free_some_pages(vcpu);
2309         r = __direct_map(vcpu, v, write, level, gfn, pfn);
2310         spin_unlock(&vcpu->kvm->mmu_lock);
2311
2312
2313         return r;
2314
2315 out_unlock:
2316         spin_unlock(&vcpu->kvm->mmu_lock);
2317         kvm_release_pfn_clean(pfn);
2318         return 0;
2319 }
2320
2321
2322 static void mmu_free_roots(struct kvm_vcpu *vcpu)
2323 {
2324         int i;
2325         struct kvm_mmu_page *sp;
2326         LIST_HEAD(invalid_list);
2327
2328         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
2329                 return;
2330         spin_lock(&vcpu->kvm->mmu_lock);
2331         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL &&
2332             (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL ||
2333              vcpu->arch.mmu.direct_map)) {
2334                 hpa_t root = vcpu->arch.mmu.root_hpa;
2335
2336                 sp = page_header(root);
2337                 --sp->root_count;
2338                 if (!sp->root_count && sp->role.invalid) {
2339                         kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
2340                         kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
2341                 }
2342                 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
2343                 spin_unlock(&vcpu->kvm->mmu_lock);
2344                 return;
2345         }
2346         for (i = 0; i < 4; ++i) {
2347                 hpa_t root = vcpu->arch.mmu.pae_root[i];
2348
2349                 if (root) {
2350                         root &= PT64_BASE_ADDR_MASK;
2351                         sp = page_header(root);
2352                         --sp->root_count;
2353                         if (!sp->root_count && sp->role.invalid)
2354                                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
2355                                                          &invalid_list);
2356                 }
2357                 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
2358         }
2359         kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
2360         spin_unlock(&vcpu->kvm->mmu_lock);
2361         vcpu->arch.mmu.root_hpa = INVALID_PAGE;
2362 }
2363
2364 static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
2365 {
2366         int ret = 0;
2367
2368         if (!kvm_is_visible_gfn(vcpu->kvm, root_gfn)) {
2369                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
2370                 ret = 1;
2371         }
2372
2373         return ret;
2374 }
2375
2376 static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu)
2377 {
2378         struct kvm_mmu_page *sp;
2379         unsigned i;
2380
2381         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
2382                 spin_lock(&vcpu->kvm->mmu_lock);
2383                 kvm_mmu_free_some_pages(vcpu);
2384                 sp = kvm_mmu_get_page(vcpu, 0, 0, PT64_ROOT_LEVEL,
2385                                       1, ACC_ALL, NULL);
2386                 ++sp->root_count;
2387                 spin_unlock(&vcpu->kvm->mmu_lock);
2388                 vcpu->arch.mmu.root_hpa = __pa(sp->spt);
2389         } else if (vcpu->arch.mmu.shadow_root_level == PT32E_ROOT_LEVEL) {
2390                 for (i = 0; i < 4; ++i) {
2391                         hpa_t root = vcpu->arch.mmu.pae_root[i];
2392
2393                         ASSERT(!VALID_PAGE(root));
2394                         spin_lock(&vcpu->kvm->mmu_lock);
2395                         kvm_mmu_free_some_pages(vcpu);
2396                         sp = kvm_mmu_get_page(vcpu, i << 30, i << 30,
2397                                               PT32_ROOT_LEVEL, 1, ACC_ALL,
2398                                               NULL);
2399                         root = __pa(sp->spt);
2400                         ++sp->root_count;
2401                         spin_unlock(&vcpu->kvm->mmu_lock);
2402                         vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
2403                 }
2404                 vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
2405         } else
2406                 BUG();
2407
2408         return 0;
2409 }
2410
2411 static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
2412 {
2413         struct kvm_mmu_page *sp;
2414         u64 pdptr, pm_mask;
2415         gfn_t root_gfn;
2416         int i;
2417
2418         root_gfn = vcpu->arch.mmu.get_cr3(vcpu) >> PAGE_SHIFT;
2419
2420         if (mmu_check_root(vcpu, root_gfn))
2421                 return 1;
2422
2423         /*
2424          * Do we shadow a long mode page table? If so we need to
2425          * write-protect the guests page table root.
2426          */
2427         if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
2428                 hpa_t root = vcpu->arch.mmu.root_hpa;
2429
2430                 ASSERT(!VALID_PAGE(root));
2431
2432                 spin_lock(&vcpu->kvm->mmu_lock);
2433                 kvm_mmu_free_some_pages(vcpu);
2434                 sp = kvm_mmu_get_page(vcpu, root_gfn, 0, PT64_ROOT_LEVEL,
2435                                       0, ACC_ALL, NULL);
2436                 root = __pa(sp->spt);
2437                 ++sp->root_count;
2438                 spin_unlock(&vcpu->kvm->mmu_lock);
2439                 vcpu->arch.mmu.root_hpa = root;
2440                 return 0;
2441         }
2442
2443         /*
2444          * We shadow a 32 bit page table. This may be a legacy 2-level
2445          * or a PAE 3-level page table. In either case we need to be aware that
2446          * the shadow page table may be a PAE or a long mode page table.
2447          */
2448         pm_mask = PT_PRESENT_MASK;
2449         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL)
2450                 pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK;
2451
2452         for (i = 0; i < 4; ++i) {
2453                 hpa_t root = vcpu->arch.mmu.pae_root[i];
2454
2455                 ASSERT(!VALID_PAGE(root));
2456                 if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
2457                         pdptr = kvm_pdptr_read_mmu(vcpu, &vcpu->arch.mmu, i);
2458                         if (!is_present_gpte(pdptr)) {
2459                                 vcpu->arch.mmu.pae_root[i] = 0;
2460                                 continue;
2461                         }
2462                         root_gfn = pdptr >> PAGE_SHIFT;
2463                         if (mmu_check_root(vcpu, root_gfn))
2464                                 return 1;
2465                 }
2466                 spin_lock(&vcpu->kvm->mmu_lock);
2467                 kvm_mmu_free_some_pages(vcpu);
2468                 sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
2469                                       PT32_ROOT_LEVEL, 0,
2470                                       ACC_ALL, NULL);
2471                 root = __pa(sp->spt);
2472                 ++sp->root_count;
2473                 spin_unlock(&vcpu->kvm->mmu_lock);
2474
2475                 vcpu->arch.mmu.pae_root[i] = root | pm_mask;
2476         }
2477         vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
2478
2479         /*
2480          * If we shadow a 32 bit page table with a long mode page
2481          * table we enter this path.
2482          */
2483         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
2484                 if (vcpu->arch.mmu.lm_root == NULL) {
2485                         /*
2486                          * The additional page necessary for this is only
2487                          * allocated on demand.
2488                          */
2489
2490                         u64 *lm_root;
2491
2492                         lm_root = (void*)get_zeroed_page(GFP_KERNEL);
2493                         if (lm_root == NULL)
2494                                 return 1;
2495
2496                         lm_root[0] = __pa(vcpu->arch.mmu.pae_root) | pm_mask;
2497
2498                         vcpu->arch.mmu.lm_root = lm_root;
2499                 }
2500
2501                 vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.lm_root);
2502         }
2503
2504         return 0;
2505 }
2506
2507 static int mmu_alloc_roots(struct kvm_vcpu *vcpu)
2508 {
2509         if (vcpu->arch.mmu.direct_map)
2510                 return mmu_alloc_direct_roots(vcpu);
2511         else
2512                 return mmu_alloc_shadow_roots(vcpu);
2513 }
2514
2515 static void mmu_sync_roots(struct kvm_vcpu *vcpu)
2516 {
2517         int i;
2518         struct kvm_mmu_page *sp;
2519
2520         if (vcpu->arch.mmu.direct_map)
2521                 return;
2522
2523         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
2524                 return;
2525
2526         trace_kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);
2527         if (vcpu->arch.mmu.root_level == PT64_ROOT_LEVEL) {
2528                 hpa_t root = vcpu->arch.mmu.root_hpa;
2529                 sp = page_header(root);
2530                 mmu_sync_children(vcpu, sp);
2531                 return;
2532         }
2533         for (i = 0; i < 4; ++i) {
2534                 hpa_t root = vcpu->arch.mmu.pae_root[i];
2535
2536                 if (root && VALID_PAGE(root)) {
2537                         root &= PT64_BASE_ADDR_MASK;
2538                         sp = page_header(root);
2539                         mmu_sync_children(vcpu, sp);
2540                 }
2541         }
2542         trace_kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
2543 }
2544
2545 void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
2546 {
2547         spin_lock(&vcpu->kvm->mmu_lock);
2548         mmu_sync_roots(vcpu);
2549         spin_unlock(&vcpu->kvm->mmu_lock);
2550 }
2551
2552 static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr,
2553                                   u32 access, u32 *error)
2554 {
2555         if (error)
2556                 *error = 0;
2557         return vaddr;
2558 }
2559
2560 static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gva_t vaddr,
2561                                          u32 access, u32 *error)
2562 {
2563         if (error)
2564                 *error = 0;
2565         return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access);
2566 }
2567
2568 static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
2569                                 u32 error_code)
2570 {
2571         gfn_t gfn;
2572         int r;
2573
2574         pgprintk("%s: gva %lx error %x\n", __func__, gva, error_code);
2575         r = mmu_topup_memory_caches(vcpu);
2576         if (r)
2577                 return r;
2578
2579         ASSERT(vcpu);
2580         ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
2581
2582         gfn = gva >> PAGE_SHIFT;
2583
2584         return nonpaging_map(vcpu, gva & PAGE_MASK,
2585                              error_code & PFERR_WRITE_MASK, gfn);
2586 }
2587
2588 static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
2589                                 u32 error_code)
2590 {
2591         pfn_t pfn;
2592         int r;
2593         int level;
2594         gfn_t gfn = gpa >> PAGE_SHIFT;
2595         unsigned long mmu_seq;
2596
2597         ASSERT(vcpu);
2598         ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
2599
2600         r = mmu_topup_memory_caches(vcpu);
2601         if (r)
2602                 return r;
2603
2604         level = mapping_level(vcpu, gfn);
2605
2606         gfn &= ~(KVM_PAGES_PER_HPAGE(level) - 1);
2607
2608         mmu_seq = vcpu->kvm->mmu_notifier_seq;
2609         smp_rmb();
2610         pfn = gfn_to_pfn(vcpu->kvm, gfn);
2611         if (is_error_pfn(pfn))
2612                 return kvm_handle_bad_page(vcpu->kvm, gfn, pfn);
2613         spin_lock(&vcpu->kvm->mmu_lock);
2614         if (mmu_notifier_retry(vcpu, mmu_seq))
2615                 goto out_unlock;
2616         kvm_mmu_free_some_pages(vcpu);
2617         r = __direct_map(vcpu, gpa, error_code & PFERR_WRITE_MASK,
2618                          level, gfn, pfn);
2619         spin_unlock(&vcpu->kvm->mmu_lock);
2620
2621         return r;
2622
2623 out_unlock:
2624         spin_unlock(&vcpu->kvm->mmu_lock);
2625         kvm_release_pfn_clean(pfn);
2626         return 0;
2627 }
2628
2629 static void nonpaging_free(struct kvm_vcpu *vcpu)
2630 {
2631         mmu_free_roots(vcpu);
2632 }
2633
2634 static int nonpaging_init_context(struct kvm_vcpu *vcpu,
2635                                   struct kvm_mmu *context)
2636 {
2637         context->new_cr3 = nonpaging_new_cr3;
2638         context->page_fault = nonpaging_page_fault;
2639         context->gva_to_gpa = nonpaging_gva_to_gpa;
2640         context->free = nonpaging_free;
2641         context->prefetch_page = nonpaging_prefetch_page;
2642         context->sync_page = nonpaging_sync_page;
2643         context->invlpg = nonpaging_invlpg;
2644         context->root_level = 0;
2645         context->shadow_root_level = PT32E_ROOT_LEVEL;
2646         context->root_hpa = INVALID_PAGE;
2647         context->direct_map = true;
2648         context->nx = false;
2649         return 0;
2650 }
2651
2652 void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
2653 {
2654         ++vcpu->stat.tlb_flush;
2655         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2656 }
2657
2658 static void paging_new_cr3(struct kvm_vcpu *vcpu)
2659 {
2660         pgprintk("%s: cr3 %lx\n", __func__, vcpu->arch.cr3);
2661         mmu_free_roots(vcpu);
2662 }
2663
2664 static unsigned long get_cr3(struct kvm_vcpu *vcpu)
2665 {
2666         return vcpu->arch.cr3;
2667 }
2668
2669 static void inject_page_fault(struct kvm_vcpu *vcpu)
2670 {
2671         vcpu->arch.mmu.inject_page_fault(vcpu);
2672 }
2673
2674 static void paging_free(struct kvm_vcpu *vcpu)
2675 {
2676         nonpaging_free(vcpu);
2677 }
2678
2679 static bool is_rsvd_bits_set(struct kvm_mmu *mmu, u64 gpte, int level)
2680 {
2681         int bit7;
2682
2683         bit7 = (gpte >> 7) & 1;
2684         return (gpte & mmu->rsvd_bits_mask[bit7][level-1]) != 0;
2685 }
2686
2687 #define PTTYPE 64
2688 #include "paging_tmpl.h"
2689 #undef PTTYPE
2690
2691 #define PTTYPE 32
2692 #include "paging_tmpl.h"
2693 #undef PTTYPE
2694
2695 static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
2696                                   struct kvm_mmu *context,
2697                                   int level)
2698 {
2699         int maxphyaddr = cpuid_maxphyaddr(vcpu);
2700         u64 exb_bit_rsvd = 0;
2701
2702         if (!context->nx)
2703                 exb_bit_rsvd = rsvd_bits(63, 63);
2704         switch (level) {
2705         case PT32_ROOT_LEVEL:
2706                 /* no rsvd bits for 2 level 4K page table entries */
2707                 context->rsvd_bits_mask[0][1] = 0;
2708                 context->rsvd_bits_mask[0][0] = 0;
2709                 context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
2710
2711                 if (!is_pse(vcpu)) {
2712                         context->rsvd_bits_mask[1][1] = 0;
2713                         break;
2714                 }
2715
2716                 if (is_cpuid_PSE36())
2717                         /* 36bits PSE 4MB page */
2718                         context->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
2719                 else
2720                         /* 32 bits PSE 4MB page */
2721                         context->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
2722                 break;
2723         case PT32E_ROOT_LEVEL:
2724                 context->rsvd_bits_mask[0][2] =
2725                         rsvd_bits(maxphyaddr, 63) |
2726                         rsvd_bits(7, 8) | rsvd_bits(1, 2);      /* PDPTE */
2727                 context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
2728                         rsvd_bits(maxphyaddr, 62);      /* PDE */
2729                 context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
2730                         rsvd_bits(maxphyaddr, 62);      /* PTE */
2731                 context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
2732                         rsvd_bits(maxphyaddr, 62) |
2733                         rsvd_bits(13, 20);              /* large page */
2734                 context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
2735                 break;
2736         case PT64_ROOT_LEVEL:
2737                 context->rsvd_bits_mask[0][3] = exb_bit_rsvd |
2738                         rsvd_bits(maxphyaddr, 51) | rsvd_bits(7, 8);
2739                 context->rsvd_bits_mask[0][2] = exb_bit_rsvd |
2740                         rsvd_bits(maxphyaddr, 51) | rsvd_bits(7, 8);
2741                 context->rsvd_bits_mask[0][1] = exb_bit_rsvd |
2742                         rsvd_bits(maxphyaddr, 51);
2743                 context->rsvd_bits_mask[0][0] = exb_bit_rsvd |
2744                         rsvd_bits(maxphyaddr, 51);
2745                 context->rsvd_bits_mask[1][3] = context->rsvd_bits_mask[0][3];
2746                 context->rsvd_bits_mask[1][2] = exb_bit_rsvd |
2747                         rsvd_bits(maxphyaddr, 51) |
2748                         rsvd_bits(13, 29);
2749                 context->rsvd_bits_mask[1][1] = exb_bit_rsvd |
2750                         rsvd_bits(maxphyaddr, 51) |
2751                         rsvd_bits(13, 20);              /* large page */
2752                 context->rsvd_bits_mask[1][0] = context->rsvd_bits_mask[0][0];
2753                 break;
2754         }
2755 }
2756
2757 static int paging64_init_context_common(struct kvm_vcpu *vcpu,
2758                                         struct kvm_mmu *context,
2759                                         int level)
2760 {
2761         context->nx = is_nx(vcpu);
2762
2763         reset_rsvds_bits_mask(vcpu, context, level);
2764
2765         ASSERT(is_pae(vcpu));
2766         context->new_cr3 = paging_new_cr3;
2767         context->page_fault = paging64_page_fault;
2768         context->gva_to_gpa = paging64_gva_to_gpa;
2769         context->prefetch_page = paging64_prefetch_page;
2770         context->sync_page = paging64_sync_page;
2771         context->invlpg = paging64_invlpg;
2772         context->free = paging_free;
2773         context->root_level = level;
2774         context->shadow_root_level = level;
2775         context->root_hpa = INVALID_PAGE;
2776         context->direct_map = false;
2777         return 0;
2778 }
2779
2780 static int paging64_init_context(struct kvm_vcpu *vcpu,
2781                                  struct kvm_mmu *context)
2782 {
2783         return paging64_init_context_common(vcpu, context, PT64_ROOT_LEVEL);
2784 }
2785
2786 static int paging32_init_context(struct kvm_vcpu *vcpu,
2787                                  struct kvm_mmu *context)
2788 {
2789         context->nx = false;
2790
2791         reset_rsvds_bits_mask(vcpu, context, PT32_ROOT_LEVEL);
2792
2793         context->new_cr3 = paging_new_cr3;
2794         context->page_fault = paging32_page_fault;
2795         context->gva_to_gpa = paging32_gva_to_gpa;
2796         context->free = paging_free;
2797         context->prefetch_page = paging32_prefetch_page;
2798         context->sync_page = paging32_sync_page;
2799         context->invlpg = paging32_invlpg;
2800         context->root_level = PT32_ROOT_LEVEL;
2801         context->shadow_root_level = PT32E_ROOT_LEVEL;
2802         context->root_hpa = INVALID_PAGE;
2803         context->direct_map = false;
2804         return 0;
2805 }
2806
2807 static int paging32E_init_context(struct kvm_vcpu *vcpu,
2808                                   struct kvm_mmu *context)
2809 {
2810         return paging64_init_context_common(vcpu, context, PT32E_ROOT_LEVEL);
2811 }
2812
2813 static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
2814 {
2815         struct kvm_mmu *context = vcpu->arch.walk_mmu;
2816
2817         context->new_cr3 = nonpaging_new_cr3;
2818         context->page_fault = tdp_page_fault;
2819         context->free = nonpaging_free;
2820         context->prefetch_page = nonpaging_prefetch_page;
2821         context->sync_page = nonpaging_sync_page;
2822         context->invlpg = nonpaging_invlpg;
2823         context->shadow_root_level = kvm_x86_ops->get_tdp_level();
2824         context->root_hpa = INVALID_PAGE;
2825         context->direct_map = true;
2826         context->set_cr3 = kvm_x86_ops->set_tdp_cr3;
2827         context->get_cr3 = get_cr3;
2828         context->inject_page_fault = kvm_inject_page_fault;
2829         context->nx = is_nx(vcpu);
2830
2831         if (!is_paging(vcpu)) {
2832                 context->nx = false;
2833                 context->gva_to_gpa = nonpaging_gva_to_gpa;
2834                 context->root_level = 0;
2835         } else if (is_long_mode(vcpu)) {
2836                 context->nx = is_nx(vcpu);
2837                 reset_rsvds_bits_mask(vcpu, context, PT64_ROOT_LEVEL);
2838                 context->gva_to_gpa = paging64_gva_to_gpa;
2839                 context->root_level = PT64_ROOT_LEVEL;
2840         } else if (is_pae(vcpu)) {
2841                 context->nx = is_nx(vcpu);
2842                 reset_rsvds_bits_mask(vcpu, context, PT32E_ROOT_LEVEL);
2843                 context->gva_to_gpa = paging64_gva_to_gpa;
2844                 context->root_level = PT32E_ROOT_LEVEL;
2845         } else {
2846                 context->nx = false;
2847                 reset_rsvds_bits_mask(vcpu, context, PT32_ROOT_LEVEL);
2848                 context->gva_to_gpa = paging32_gva_to_gpa;
2849                 context->root_level = PT32_ROOT_LEVEL;
2850         }
2851
2852         return 0;
2853 }
2854
2855 int kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *context)
2856 {
2857         int r;
2858         ASSERT(vcpu);
2859         ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
2860
2861         if (!is_paging(vcpu))
2862                 r = nonpaging_init_context(vcpu, context);
2863         else if (is_long_mode(vcpu))
2864                 r = paging64_init_context(vcpu, context);
2865         else if (is_pae(vcpu))
2866                 r = paging32E_init_context(vcpu, context);
2867         else
2868                 r = paging32_init_context(vcpu, context);
2869
2870         vcpu->arch.mmu.base_role.cr4_pae = !!is_pae(vcpu);
2871         vcpu->arch.mmu.base_role.cr0_wp  = is_write_protection(vcpu);
2872
2873         return r;
2874 }
2875 EXPORT_SYMBOL_GPL(kvm_init_shadow_mmu);
2876
2877 static int init_kvm_softmmu(struct kvm_vcpu *vcpu)
2878 {
2879         int r = kvm_init_shadow_mmu(vcpu, vcpu->arch.walk_mmu);
2880
2881         vcpu->arch.walk_mmu->set_cr3           = kvm_x86_ops->set_cr3;
2882         vcpu->arch.walk_mmu->get_cr3           = get_cr3;
2883         vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault;
2884
2885         return r;
2886 }
2887
2888 static int init_kvm_nested_mmu(struct kvm_vcpu *vcpu)
2889 {
2890         struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;
2891
2892         g_context->get_cr3           = get_cr3;
2893         g_context->inject_page_fault = kvm_inject_page_fault;
2894
2895         /*
2896          * Note that arch.mmu.gva_to_gpa translates l2_gva to l1_gpa. The
2897          * translation of l2_gpa to l1_gpa addresses is done using the
2898          * arch.nested_mmu.gva_to_gpa function. Basically the gva_to_gpa
2899          * functions between mmu and nested_mmu are swapped.
2900          */
2901         if (!is_paging(vcpu)) {
2902                 g_context->nx = false;
2903                 g_context->root_level = 0;
2904                 g_context->gva_to_gpa = nonpaging_gva_to_gpa_nested;
2905         } else if (is_long_mode(vcpu)) {
2906                 g_context->nx = is_nx(vcpu);
2907                 reset_rsvds_bits_mask(vcpu, g_context, PT64_ROOT_LEVEL);
2908                 g_context->root_level = PT64_ROOT_LEVEL;
2909                 g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
2910         } else if (is_pae(vcpu)) {
2911                 g_context->nx = is_nx(vcpu);
2912                 reset_rsvds_bits_mask(vcpu, g_context, PT32E_ROOT_LEVEL);
2913                 g_context->root_level = PT32E_ROOT_LEVEL;
2914                 g_context->gva_to_gpa = paging64_gva_to_gpa_nested;
2915         } else {
2916                 g_context->nx = false;
2917                 reset_rsvds_bits_mask(vcpu, g_context, PT32_ROOT_LEVEL);
2918                 g_context->root_level = PT32_ROOT_LEVEL;
2919                 g_context->gva_to_gpa = paging32_gva_to_gpa_nested;
2920         }
2921
2922         return 0;
2923 }
2924
2925 static int init_kvm_mmu(struct kvm_vcpu *vcpu)
2926 {
2927         vcpu->arch.update_pte.pfn = bad_pfn;
2928
2929         if (mmu_is_nested(vcpu))
2930                 return init_kvm_nested_mmu(vcpu);
2931         else if (tdp_enabled)
2932                 return init_kvm_tdp_mmu(vcpu);
2933         else
2934                 return init_kvm_softmmu(vcpu);
2935 }
2936
2937 static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
2938 {
2939         ASSERT(vcpu);
2940         if (VALID_PAGE(vcpu->arch.mmu.root_hpa))
2941                 /* mmu.free() should set root_hpa = INVALID_PAGE */
2942                 vcpu->arch.mmu.free(vcpu);
2943 }
2944
2945 int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
2946 {
2947         destroy_kvm_mmu(vcpu);
2948         return init_kvm_mmu(vcpu);
2949 }
2950 EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
2951
2952 int kvm_mmu_load(struct kvm_vcpu *vcpu)
2953 {
2954         int r;
2955
2956         r = mmu_topup_memory_caches(vcpu);
2957         if (r)
2958                 goto out;
2959         r = mmu_alloc_roots(vcpu);
2960         spin_lock(&vcpu->kvm->mmu_lock);
2961         mmu_sync_roots(vcpu);
2962         spin_unlock(&vcpu->kvm->mmu_lock);
2963         if (r)
2964                 goto out;
2965         /* set_cr3() should ensure TLB has been flushed */
2966         vcpu->arch.mmu.set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
2967 out:
2968         return r;
2969 }
2970 EXPORT_SYMBOL_GPL(kvm_mmu_load);
2971
2972 void kvm_mmu_unload(struct kvm_vcpu *vcpu)
2973 {
2974         mmu_free_roots(vcpu);
2975 }
2976 EXPORT_SYMBOL_GPL(kvm_mmu_unload);
2977
2978 static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu,
2979                                   struct kvm_mmu_page *sp,
2980                                   u64 *spte)
2981 {
2982         u64 pte;
2983         struct kvm_mmu_page *child;
2984
2985         pte = *spte;
2986         if (is_shadow_present_pte(pte)) {
2987                 if (is_last_spte(pte, sp->role.level))
2988                         drop_spte(vcpu->kvm, spte, shadow_trap_nonpresent_pte);
2989                 else {
2990                         child = page_header(pte & PT64_BASE_ADDR_MASK);
2991                         mmu_page_remove_parent_pte(child, spte);
2992                 }
2993         }
2994         __set_spte(spte, shadow_trap_nonpresent_pte);
2995         if (is_large_pte(pte))
2996                 --vcpu->kvm->stat.lpages;
2997 }
2998
2999 static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
3000                                   struct kvm_mmu_page *sp,
3001                                   u64 *spte,
3002                                   const void *new)
3003 {
3004         if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
3005                 ++vcpu->kvm->stat.mmu_pde_zapped;
3006                 return;
3007         }
3008
3009         if (is_rsvd_bits_set(&vcpu->arch.mmu, *(u64 *)new, PT_PAGE_TABLE_LEVEL))
3010                 return;
3011
3012         ++vcpu->kvm->stat.mmu_pte_updated;
3013         if (!sp->role.cr4_pae)
3014                 paging32_update_pte(vcpu, sp, spte, new);
3015         else
3016                 paging64_update_pte(vcpu, sp, spte, new);
3017 }
3018
3019 static bool need_remote_flush(u64 old, u64 new)
3020 {
3021         if (!is_shadow_present_pte(old))
3022                 return false;
3023         if (!is_shadow_present_pte(new))
3024                 return true;
3025         if ((old ^ new) & PT64_BASE_ADDR_MASK)
3026                 return true;
3027         old ^= PT64_NX_MASK;
3028         new ^= PT64_NX_MASK;
3029         return (old & ~new & PT64_PERM_MASK) != 0;
3030 }
3031
3032 static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, bool zap_page,
3033                                     bool remote_flush, bool local_flush)
3034 {
3035         if (zap_page)
3036                 return;
3037
3038         if (remote_flush)
3039                 kvm_flush_remote_tlbs(vcpu->kvm);
3040         else if (local_flush)
3041                 kvm_mmu_flush_tlb(vcpu);
3042 }
3043
3044 static bool last_updated_pte_accessed(struct kvm_vcpu *vcpu)
3045 {
3046         u64 *spte = vcpu->arch.last_pte_updated;
3047
3048         return !!(spte && (*spte & shadow_accessed_mask));
3049 }
3050
3051 static void mmu_guess_page_from_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
3052                                           u64 gpte)
3053 {
3054         gfn_t gfn;
3055         pfn_t pfn;
3056
3057         if (!is_present_gpte(gpte))
3058                 return;
3059         gfn = (gpte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
3060
3061         vcpu->arch.update_pte.mmu_seq = vcpu->kvm->mmu_notifier_seq;
3062         smp_rmb();
3063         pfn = gfn_to_pfn(vcpu->kvm, gfn);
3064
3065         if (is_error_pfn(pfn)) {
3066                 kvm_release_pfn_clean(pfn);
3067                 return;
3068         }
3069         vcpu->arch.update_pte.gfn = gfn;
3070         vcpu->arch.update_pte.pfn = pfn;
3071 }
3072
3073 static void kvm_mmu_access_page(struct kvm_vcpu *vcpu, gfn_t gfn)
3074 {
3075         u64 *spte = vcpu->arch.last_pte_updated;
3076
3077         if (spte
3078             && vcpu->arch.last_pte_gfn == gfn
3079             && shadow_accessed_mask
3080             && !(*spte & shadow_accessed_mask)
3081             && is_shadow_present_pte(*spte))
3082                 set_bit(PT_ACCESSED_SHIFT, (unsigned long *)spte);
3083 }
3084
3085 void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
3086                        const u8 *new, int bytes,
3087                        bool guest_initiated)
3088 {
3089         gfn_t gfn = gpa >> PAGE_SHIFT;
3090         union kvm_mmu_page_role mask = { .word = 0 };
3091         struct kvm_mmu_page *sp;
3092         struct hlist_node *node;
3093         LIST_HEAD(invalid_list);
3094         u64 entry, gentry;
3095         u64 *spte;
3096         unsigned offset = offset_in_page(gpa);
3097         unsigned pte_size;
3098         unsigned page_offset;
3099         unsigned misaligned;
3100         unsigned quadrant;
3101         int level;
3102         int flooded = 0;
3103         int npte;
3104         int r;
3105         int invlpg_counter;
3106         bool remote_flush, local_flush, zap_page;
3107
3108         zap_page = remote_flush = local_flush = false;
3109
3110         pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);
3111
3112         invlpg_counter = atomic_read(&vcpu->kvm->arch.invlpg_counter);
3113
3114         /*
3115          * Assume that the pte write on a page table of the same type
3116          * as the current vcpu paging mode.  This is nearly always true
3117          * (might be false while changing modes).  Note it is verified later
3118          * by update_pte().
3119          */
3120         if ((is_pae(vcpu) && bytes == 4) || !new) {
3121                 /* Handle a 32-bit guest writing two halves of a 64-bit gpte */
3122                 if (is_pae(vcpu)) {
3123                         gpa &= ~(gpa_t)7;
3124                         bytes = 8;
3125                 }
3126                 r = kvm_read_guest(vcpu->kvm, gpa, &gentry, min(bytes, 8));
3127                 if (r)
3128                         gentry = 0;
3129                 new = (const u8 *)&gentry;
3130         }
3131
3132         switch (bytes) {
3133         case 4:
3134                 gentry = *(const u32 *)new;
3135                 break;
3136         case 8:
3137                 gentry = *(const u64 *)new;
3138                 break;
3139         default:
3140                 gentry = 0;
3141                 break;
3142         }
3143
3144         mmu_guess_page_from_pte_write(vcpu, gpa, gentry);
3145         spin_lock(&vcpu->kvm->mmu_lock);
3146         if (atomic_read(&vcpu->kvm->arch.invlpg_counter) != invlpg_counter)
3147                 gentry = 0;
3148         kvm_mmu_access_page(vcpu, gfn);
3149         kvm_mmu_free_some_pages(vcpu);
3150         ++vcpu->kvm->stat.mmu_pte_write;
3151         trace_kvm_mmu_audit(vcpu, AUDIT_PRE_PTE_WRITE);
3152         if (guest_initiated) {
3153                 if (gfn == vcpu->arch.last_pt_write_gfn
3154                     && !last_updated_pte_accessed(vcpu)) {
3155                         ++vcpu->arch.last_pt_write_count;
3156                         if (vcpu->arch.last_pt_write_count >= 3)
3157                                 flooded = 1;
3158                 } else {
3159                         vcpu->arch.last_pt_write_gfn = gfn;
3160                         vcpu->arch.last_pt_write_count = 1;
3161                         vcpu->arch.last_pte_updated = NULL;
3162                 }
3163         }
3164
3165         mask.cr0_wp = mask.cr4_pae = mask.nxe = 1;
3166         for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn, node) {
3167                 pte_size = sp->role.cr4_pae ? 8 : 4;
3168                 misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
3169                 misaligned |= bytes < 4;
3170                 if (misaligned || flooded) {
3171                         /*
3172                          * Misaligned accesses are too much trouble to fix
3173                          * up; also, they usually indicate a page is not used
3174                          * as a page table.
3175                          *
3176                          * If we're seeing too many writes to a page,
3177                          * it may no longer be a page table, or we may be
3178                          * forking, in which case it is better to unmap the
3179                          * page.
3180                          */
3181                         pgprintk("misaligned: gpa %llx bytes %d role %x\n",
3182                                  gpa, bytes, sp->role.word);
3183                         zap_page |= !!kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
3184                                                      &invalid_list);
3185                         ++vcpu->kvm->stat.mmu_flooded;
3186                         continue;
3187                 }
3188                 page_offset = offset;
3189                 level = sp->role.level;
3190                 npte = 1;
3191                 if (!sp->role.cr4_pae) {
3192                         page_offset <<= 1;      /* 32->64 */
3193                         /*
3194                          * A 32-bit pde maps 4MB while the shadow pdes map
3195                          * only 2MB.  So we need to double the offset again
3196                          * and zap two pdes instead of one.
3197                          */
3198                         if (level == PT32_ROOT_LEVEL) {
3199                                 page_offset &= ~7; /* kill rounding error */
3200                                 page_offset <<= 1;
3201                                 npte = 2;
3202                         }
3203                         quadrant = page_offset >> PAGE_SHIFT;
3204                         page_offset &= ~PAGE_MASK;
3205                         if (quadrant != sp->role.quadrant)
3206                                 continue;
3207                 }
3208                 local_flush = true;
3209                 spte = &sp->spt[page_offset / sizeof(*spte)];
3210                 while (npte--) {
3211                         entry = *spte;
3212                         mmu_pte_write_zap_pte(vcpu, sp, spte);
3213                         if (gentry &&
3214                               !((sp->role.word ^ vcpu->arch.mmu.base_role.word)
3215                               & mask.word))
3216                                 mmu_pte_write_new_pte(vcpu, sp, spte, &gentry);
3217                         if (!remote_flush && need_remote_flush(entry, *spte))
3218                                 remote_flush = true;
3219                         ++spte;
3220                 }
3221         }
3222         mmu_pte_write_flush_tlb(vcpu, zap_page, remote_flush, local_flush);
3223         kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
3224         trace_kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE);
3225         spin_unlock(&vcpu->kvm->mmu_lock);
3226         if (!is_error_pfn(vcpu->arch.update_pte.pfn)) {
3227                 kvm_release_pfn_clean(vcpu->arch.update_pte.pfn);
3228                 vcpu->arch.update_pte.pfn = bad_pfn;
3229         }
3230 }
3231
3232 int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
3233 {
3234         gpa_t gpa;
3235         int r;
3236
3237         if (vcpu->arch.mmu.direct_map)
3238                 return 0;
3239
3240         gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL);
3241
3242         spin_lock(&vcpu->kvm->mmu_lock);
3243         r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
3244         spin_unlock(&vcpu->kvm->mmu_lock);
3245         return r;
3246 }
3247 EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt);
3248
3249 void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
3250 {
3251         LIST_HEAD(invalid_list);
3252
3253         while (kvm_mmu_available_pages(vcpu->kvm) < KVM_REFILL_PAGES &&
3254                !list_empty(&vcpu->kvm->arch.active_mmu_pages)) {
3255                 struct kvm_mmu_page *sp;
3256
3257                 sp = container_of(vcpu->kvm->arch.active_mmu_pages.prev,
3258                                   struct kvm_mmu_page, link);
3259                 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
3260                 kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
3261                 ++vcpu->kvm->stat.mmu_recycled;
3262         }
3263 }
3264
3265 int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code)
3266 {
3267         int r;
3268         enum emulation_result er;
3269
3270         r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code);
3271         if (r < 0)
3272                 goto out;
3273
3274         if (!r) {
3275                 r = 1;
3276                 goto out;
3277         }
3278
3279         r = mmu_topup_memory_caches(vcpu);
3280         if (r)
3281                 goto out;
3282
3283         er = emulate_instruction(vcpu, cr2, error_code, 0);
3284
3285         switch (er) {
3286         case EMULATE_DONE:
3287                 return 1;
3288         case EMULATE_DO_MMIO:
3289                 ++vcpu->stat.mmio_exits;
3290                 /* fall through */
3291         case EMULATE_FAIL:
3292                 return 0;
3293         default:
3294                 BUG();
3295         }
3296 out:
3297         return r;
3298 }
3299 EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
3300
3301 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
3302 {
3303         vcpu->arch.mmu.invlpg(vcpu, gva);
3304         kvm_mmu_flush_tlb(vcpu);
3305         ++vcpu->stat.invlpg;
3306 }
3307 EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);
3308
3309 void kvm_enable_tdp(void)
3310 {
3311         tdp_enabled = true;
3312 }
3313 EXPORT_SYMBOL_GPL(kvm_enable_tdp);
3314
3315 void kvm_disable_tdp(void)
3316 {
3317         tdp_enabled = false;
3318 }
3319 EXPORT_SYMBOL_GPL(kvm_disable_tdp);
3320
3321 static void free_mmu_pages(struct kvm_vcpu *vcpu)
3322 {
3323         free_page((unsigned long)vcpu->arch.mmu.pae_root);
3324         if (vcpu->arch.mmu.lm_root != NULL)
3325                 free_page((unsigned long)vcpu->arch.mmu.lm_root);
3326 }
3327
3328 static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
3329 {
3330         struct page *page;
3331         int i;
3332
3333         ASSERT(vcpu);
3334
3335         /*
3336          * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
3337          * Therefore we need to allocate shadow page tables in the first
3338          * 4GB of memory, which happens to fit the DMA32 zone.
3339          */
3340         page = alloc_page(GFP_KERNEL | __GFP_DMA32);
3341         if (!page)
3342                 return -ENOMEM;
3343
3344         vcpu->arch.mmu.pae_root = page_address(page);
3345         for (i = 0; i < 4; ++i)
3346                 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
3347
3348         return 0;
3349 }
3350
3351 int kvm_mmu_create(struct kvm_vcpu *vcpu)
3352 {
3353         ASSERT(vcpu);
3354         ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
3355
3356         return alloc_mmu_pages(vcpu);
3357 }
3358
3359 int kvm_mmu_setup(struct kvm_vcpu *vcpu)
3360 {
3361         ASSERT(vcpu);
3362         ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
3363
3364         return init_kvm_mmu(vcpu);
3365 }
3366
3367 void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
3368 {
3369         struct kvm_mmu_page *sp;
3370
3371         list_for_each_entry(sp, &kvm->arch.active_mmu_pages, link) {
3372                 int i;
3373                 u64 *pt;
3374
3375                 if (!test_bit(slot, sp->slot_bitmap))
3376                         continue;
3377
3378                 pt = sp->spt;
3379                 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
3380                         /* avoid RMW */
3381                         if (is_writable_pte(pt[i]))
3382                                 pt[i] &= ~PT_WRITABLE_MASK;
3383         }
3384         kvm_flush_remote_tlbs(kvm);
3385 }
3386
3387 void kvm_mmu_zap_all(struct kvm *kvm)
3388 {
3389         struct kvm_mmu_page *sp, *node;
3390         LIST_HEAD(invalid_list);
3391
3392         spin_lock(&kvm->mmu_lock);
3393 restart:
3394         list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link)
3395                 if (kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list))
3396                         goto restart;
3397
3398         kvm_mmu_commit_zap_page(kvm, &invalid_list);
3399         spin_unlock(&kvm->mmu_lock);
3400 }
3401
3402 static int kvm_mmu_remove_some_alloc_mmu_pages(struct kvm *kvm,
3403                                                struct list_head *invalid_list)
3404 {
3405         struct kvm_mmu_page *page;
3406
3407         page = container_of(kvm->arch.active_mmu_pages.prev,
3408                             struct kvm_mmu_page, link);
3409         return kvm_mmu_prepare_zap_page(kvm, page, invalid_list);
3410 }
3411
3412 static int mmu_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
3413 {
3414         struct kvm *kvm;
3415         struct kvm *kvm_freed = NULL;
3416
3417         if (nr_to_scan == 0)
3418                 goto out;
3419
3420         spin_lock(&kvm_lock);
3421
3422         list_for_each_entry(kvm, &vm_list, vm_list) {
3423                 int idx, freed_pages;
3424                 LIST_HEAD(invalid_list);
3425
3426                 idx = srcu_read_lock(&kvm->srcu);
3427                 spin_lock(&kvm->mmu_lock);
3428                 if (!kvm_freed && nr_to_scan > 0 &&
3429                     kvm->arch.n_used_mmu_pages > 0) {
3430                         freed_pages = kvm_mmu_remove_some_alloc_mmu_pages(kvm,
3431                                                           &invalid_list);
3432                         kvm_freed = kvm;
3433                 }
3434                 nr_to_scan--;
3435
3436                 kvm_mmu_commit_zap_page(kvm, &invalid_list);
3437                 spin_unlock(&kvm->mmu_lock);
3438                 srcu_read_unlock(&kvm->srcu, idx);
3439         }
3440         if (kvm_freed)
3441                 list_move_tail(&kvm_freed->vm_list, &vm_list);
3442
3443         spin_unlock(&kvm_lock);
3444
3445 out:
3446         return percpu_counter_read_positive(&kvm_total_used_mmu_pages);
3447 }
3448
3449 static struct shrinker mmu_shrinker = {
3450         .shrink = mmu_shrink,
3451         .seeks = DEFAULT_SEEKS * 10,
3452 };
3453
3454 static void mmu_destroy_caches(void)
3455 {
3456         if (pte_chain_cache)
3457                 kmem_cache_destroy(pte_chain_cache);
3458         if (rmap_desc_cache)
3459                 kmem_cache_destroy(rmap_desc_cache);
3460         if (mmu_page_header_cache)
3461                 kmem_cache_destroy(mmu_page_header_cache);
3462 }
3463
3464 void kvm_mmu_module_exit(void)
3465 {
3466         mmu_destroy_caches();
3467         percpu_counter_destroy(&kvm_total_used_mmu_pages);
3468         unregister_shrinker(&mmu_shrinker);
3469 }
3470
3471 int kvm_mmu_module_init(void)
3472 {
3473         pte_chain_cache = kmem_cache_create("kvm_pte_chain",
3474                                             sizeof(struct kvm_pte_chain),
3475                                             0, 0, NULL);
3476         if (!pte_chain_cache)
3477                 goto nomem;
3478         rmap_desc_cache = kmem_cache_create("kvm_rmap_desc",
3479                                             sizeof(struct kvm_rmap_desc),
3480                                             0, 0, NULL);
3481         if (!rmap_desc_cache)
3482                 goto nomem;
3483
3484         mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
3485                                                   sizeof(struct kvm_mmu_page),
3486                                                   0, 0, NULL);
3487         if (!mmu_page_header_cache)
3488                 goto nomem;
3489
3490         if (percpu_counter_init(&kvm_total_used_mmu_pages, 0))
3491                 goto nomem;
3492
3493         register_shrinker(&mmu_shrinker);
3494
3495         return 0;
3496
3497 nomem:
3498         mmu_destroy_caches();
3499         return -ENOMEM;
3500 }
3501
3502 /*
3503  * Caculate mmu pages needed for kvm.
3504  */
3505 unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
3506 {
3507         int i;
3508         unsigned int nr_mmu_pages;
3509         unsigned int  nr_pages = 0;
3510         struct kvm_memslots *slots;
3511
3512         slots = kvm_memslots(kvm);
3513
3514         for (i = 0; i < slots->nmemslots; i++)
3515                 nr_pages += slots->memslots[i].npages;
3516
3517         nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
3518         nr_mmu_pages = max(nr_mmu_pages,
3519                         (unsigned int) KVM_MIN_ALLOC_MMU_PAGES);
3520
3521         return nr_mmu_pages;
3522 }
3523
3524 static void *pv_mmu_peek_buffer(struct kvm_pv_mmu_op_buffer *buffer,
3525                                 unsigned len)
3526 {
3527         if (len > buffer->len)
3528                 return NULL;
3529         return buffer->ptr;
3530 }
3531
3532 static void *pv_mmu_read_buffer(struct kvm_pv_mmu_op_buffer *buffer,
3533                                 unsigned len)
3534 {
3535         void *ret;
3536
3537         ret = pv_mmu_peek_buffer(buffer, len);
3538         if (!ret)
3539                 return ret;
3540         buffer->ptr += len;
3541         buffer->len -= len;
3542         buffer->processed += len;
3543         return ret;
3544 }
3545
3546 static int kvm_pv_mmu_write(struct kvm_vcpu *vcpu,
3547                              gpa_t addr, gpa_t value)
3548 {
3549         int bytes = 8;
3550         int r;
3551
3552         if (!is_long_mode(vcpu) && !is_pae(vcpu))
3553                 bytes = 4;
3554
3555         r = mmu_topup_memory_caches(vcpu);
3556         if (r)
3557                 return r;
3558
3559         if (!emulator_write_phys(vcpu, addr, &value, bytes))
3560                 return -EFAULT;
3561
3562         return 1;
3563 }
3564
3565 static int kvm_pv_mmu_flush_tlb(struct kvm_vcpu *vcpu)
3566 {
3567         (void)kvm_set_cr3(vcpu, vcpu->arch.cr3);
3568         return 1;
3569 }
3570
3571 static int kvm_pv_mmu_release_pt(struct kvm_vcpu *vcpu, gpa_t addr)
3572 {
3573         spin_lock(&vcpu->kvm->mmu_lock);
3574         mmu_unshadow(vcpu->kvm, addr >> PAGE_SHIFT);
3575         spin_unlock(&vcpu->kvm->mmu_lock);
3576         return 1;
3577 }
3578
3579 static int kvm_pv_mmu_op_one(struct kvm_vcpu *vcpu,
3580                              struct kvm_pv_mmu_op_buffer *buffer)
3581 {
3582         struct kvm_mmu_op_header *header;
3583
3584         header = pv_mmu_peek_buffer(buffer, sizeof *header);
3585         if (!header)
3586                 return 0;
3587         switch (header->op) {
3588         case KVM_MMU_OP_WRITE_PTE: {
3589                 struct kvm_mmu_op_write_pte *wpte;
3590
3591                 wpte = pv_mmu_read_buffer(buffer, sizeof *wpte);
3592                 if (!wpte)
3593                         return 0;
3594                 return kvm_pv_mmu_write(vcpu, wpte->pte_phys,
3595                                         wpte->pte_val);
3596         }
3597         case KVM_MMU_OP_FLUSH_TLB: {
3598                 struct kvm_mmu_op_flush_tlb *ftlb;
3599
3600                 ftlb = pv_mmu_read_buffer(buffer, sizeof *ftlb);
3601                 if (!ftlb)
3602                         return 0;
3603                 return kvm_pv_mmu_flush_tlb(vcpu);
3604         }
3605         case KVM_MMU_OP_RELEASE_PT: {
3606                 struct kvm_mmu_op_release_pt *rpt;
3607
3608                 rpt = pv_mmu_read_buffer(buffer, sizeof *rpt);
3609                 if (!rpt)
3610                         return 0;
3611                 return kvm_pv_mmu_release_pt(vcpu, rpt->pt_phys);
3612         }
3613         default: return 0;
3614         }
3615 }
3616
3617 int kvm_pv_mmu_op(struct kvm_vcpu *vcpu, unsigned long bytes,
3618                   gpa_t addr, unsigned long *ret)
3619 {
3620         int r;
3621         struct kvm_pv_mmu_op_buffer *buffer = &vcpu->arch.mmu_op_buffer;
3622
3623         buffer->ptr = buffer->buf;
3624         buffer->len = min_t(unsigned long, bytes, sizeof buffer->buf);
3625         buffer->processed = 0;
3626
3627         r = kvm_read_guest(vcpu->kvm, addr, buffer->buf, buffer->len);
3628         if (r)
3629                 goto out;
3630
3631         while (buffer->len) {
3632                 r = kvm_pv_mmu_op_one(vcpu, buffer);
3633                 if (r < 0)
3634                         goto out;
3635                 if (r == 0)
3636                         break;
3637         }
3638
3639         r = 1;
3640 out:
3641         *ret = buffer->processed;
3642         return r;
3643 }
3644
3645 int kvm_mmu_get_spte_hierarchy(struct kvm_vcpu *vcpu, u64 addr, u64 sptes[4])
3646 {
3647         struct kvm_shadow_walk_iterator iterator;
3648         int nr_sptes = 0;
3649
3650         spin_lock(&vcpu->kvm->mmu_lock);
3651         for_each_shadow_entry(vcpu, addr, iterator) {
3652                 sptes[iterator.level-1] = *iterator.sptep;
3653                 nr_sptes++;
3654                 if (!is_shadow_present_pte(*iterator.sptep))
3655                         break;
3656         }
3657         spin_unlock(&vcpu->kvm->mmu_lock);
3658
3659         return nr_sptes;
3660 }
3661 EXPORT_SYMBOL_GPL(kvm_mmu_get_spte_hierarchy);
3662
3663 #ifdef CONFIG_KVM_MMU_AUDIT
3664 #include "mmu_audit.c"
3665 #else
3666 static void mmu_audit_disable(void) { }
3667 #endif
3668
3669 void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
3670 {
3671         ASSERT(vcpu);
3672
3673         destroy_kvm_mmu(vcpu);
3674         free_mmu_pages(vcpu);
3675         mmu_free_memory_caches(vcpu);
3676         mmu_audit_disable();
3677 }