drivers/iommu/intel-iommu.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  * Author: Fenghua Yu <fenghua.yu@intel.com>
  22  */
  23
  24 #include <linux/init.h>
  25 #include <linux/bitmap.h>
  26 #include <linux/debugfs.h>
  27 #include <linux/export.h>
  28 #include <linux/slab.h>
  29 #include <linux/irq.h>
  30 #include <linux/interrupt.h>
  31 #include <linux/spinlock.h>
  32 #include <linux/pci.h>
  33 #include <linux/dmar.h>
  34 #include <linux/dma-mapping.h>
  35 #include <linux/mempool.h>
  36 #include <linux/timer.h>
  37 #include <linux/iova.h>
  38 #include <linux/iommu.h>
  39 #include <linux/intel-iommu.h>
  40 #include <linux/syscore_ops.h>
  41 #include <linux/tboot.h>
  42 #include <linux/dmi.h>
  43 #include <linux/pci-ats.h>
  44 #include <asm/cacheflush.h>
  45 #include <asm/iommu.h>
  46
  47 #define ROOT_SIZE               VTD_PAGE_SIZE
  48 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  49
  50 #define IS_BRIDGE_HOST_DEVICE(pdev) \
  51                             ((pdev->class >> 8) == PCI_CLASS_BRIDGE_HOST)
  52 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  53 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  54 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
  55
  56 #define IOAPIC_RANGE_START      (0xfee00000)
  57 #define IOAPIC_RANGE_END        (0xfeefffff)
  58 #define IOVA_START_ADDR         (0x1000)
  59
  60 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
  61
  62 #define MAX_AGAW_WIDTH 64
  63
  64 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
  65 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
  66
  67 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
  68    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
  69 #define DOMAIN_MAX_PFN(gaw)     ((unsigned long) min_t(uint64_t, \
  70                                 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
  71 #define DOMAIN_MAX_ADDR(gaw)    (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
  72
  73 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  74 #define DMA_32BIT_PFN           IOVA_PFN(DMA_BIT_MASK(32))
  75 #define DMA_64BIT_PFN           IOVA_PFN(DMA_BIT_MASK(64))
  76
  77 /* page table handling */
  78 #define LEVEL_STRIDE            (9)
  79 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
  80
  81 static inline int agaw_to_level(int agaw)
  82 {
  83         return agaw + 2;
  84 }
  85
  86 static inline int agaw_to_width(int agaw)
  87 {
  88         return 30 + agaw * LEVEL_STRIDE;
  89 }
  90
  91 static inline int width_to_agaw(int width)
  92 {
  93         return (width - 30) / LEVEL_STRIDE;
  94 }
  95
  96 static inline unsigned int level_to_offset_bits(int level)
  97 {
  98         return (level - 1) * LEVEL_STRIDE;
  99 }
 100
 101 static inline int pfn_level_offset(unsigned long pfn, int level)
 102 {
 103         return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
 104 }
 105
 106 static inline unsigned long level_mask(int level)
 107 {
 108         return -1UL << level_to_offset_bits(level);
 109 }
 110
 111 static inline unsigned long level_size(int level)
 112 {
 113         return 1UL << level_to_offset_bits(level);
 114 }
 115
 116 static inline unsigned long align_to_level(unsigned long pfn, int level)
 117 {
 118         return (pfn + level_size(level) - 1) & level_mask(level);
 119 }
 120
 121 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
 122 {
 123         return  1 << ((lvl - 1) * LEVEL_STRIDE);
 124 }
 125
 126 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
 127    are never going to work. */
 128 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
 129 {
 130         return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
 131 }
 132
 133 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
 134 {
 135         return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
 136 }
 137 static inline unsigned long page_to_dma_pfn(struct page *pg)
 138 {
 139         return mm_to_dma_pfn(page_to_pfn(pg));
 140 }
 141 static inline unsigned long virt_to_dma_pfn(void *p)
 142 {
 143         return page_to_dma_pfn(virt_to_page(p));
 144 }
 145
 146 /* global iommu list, set NULL for ignored DMAR units */
 147 static struct intel_iommu **g_iommus;
 148
 149 static void __init check_tylersburg_isoch(void);
 150 static int rwbf_quirk;
 151
 152 /*
 153  * set to 1 to panic kernel if can't successfully enable VT-d
 154  * (used when kernel is launched w/ TXT)
 155  */
 156 static int force_on = 0;
 157
 158 /*
 159  * 0: Present
 160  * 1-11: Reserved
 161  * 12-63: Context Ptr (12 - (haw-1))
 162  * 64-127: Reserved
 163  */
 164 struct root_entry {
 165         u64     val;
 166         u64     rsvd1;
 167 };
 168 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
 169 static inline bool root_present(struct root_entry *root)
 170 {
 171         return (root->val & 1);
 172 }
 173 static inline void set_root_present(struct root_entry *root)
 174 {
 175         root->val |= 1;
 176 }
 177 static inline void set_root_value(struct root_entry *root, unsigned long value)
 178 {
 179         root->val |= value & VTD_PAGE_MASK;
 180 }
 181
 182 static inline struct context_entry *
 183 get_context_addr_from_root(struct root_entry *root)
 184 {
 185         return (struct context_entry *)
 186                 (root_present(root)?phys_to_virt(
 187                 root->val & VTD_PAGE_MASK) :
 188                 NULL);
 189 }
 190
 191 /*
 192  * low 64 bits:
 193  * 0: present
 194  * 1: fault processing disable
 195  * 2-3: translation type
 196  * 12-63: address space root
 197  * high 64 bits:
 198  * 0-2: address width
 199  * 3-6: aval
 200  * 8-23: domain id
 201  */
 202 struct context_entry {
 203         u64 lo;
 204         u64 hi;
 205 };
 206
 207 static inline bool context_present(struct context_entry *context)
 208 {
 209         return (context->lo & 1);
 210 }
 211 static inline void context_set_present(struct context_entry *context)
 212 {
 213         context->lo |= 1;
 214 }
 215
 216 static inline void context_set_fault_enable(struct context_entry *context)
 217 {
 218         context->lo &= (((u64)-1) << 2) | 1;
 219 }
 220
 221 static inline void context_set_translation_type(struct context_entry *context,
 222                                                 unsigned long value)
 223 {
 224         context->lo &= (((u64)-1) << 4) | 3;
 225         context->lo |= (value & 3) << 2;
 226 }
 227
 228 static inline void context_set_address_root(struct context_entry *context,
 229                                             unsigned long value)
 230 {
 231         context->lo |= value & VTD_PAGE_MASK;
 232 }
 233
 234 static inline void context_set_address_width(struct context_entry *context,
 235                                              unsigned long value)
 236 {
 237         context->hi |= value & 7;
 238 }
 239
 240 static inline void context_set_domain_id(struct context_entry *context,
 241                                          unsigned long value)
 242 {
 243         context->hi |= (value & ((1 << 16) - 1)) << 8;
 244 }
 245
 246 static inline void context_clear_entry(struct context_entry *context)
 247 {
 248         context->lo = 0;
 249         context->hi = 0;
 250 }
 251
 252 /*
 253  * 0: readable
 254  * 1: writable
 255  * 2-6: reserved
 256  * 7: super page
 257  * 8-10: available
 258  * 11: snoop behavior
 259  * 12-63: Host physcial address
 260  */
 261 struct dma_pte {
 262         u64 val;
 263 };
 264
 265 static inline void dma_clear_pte(struct dma_pte *pte)
 266 {
 267         pte->val = 0;
 268 }
 269
 270 static inline void dma_set_pte_readable(struct dma_pte *pte)
 271 {
 272         pte->val |= DMA_PTE_READ;
 273 }
 274
 275 static inline void dma_set_pte_writable(struct dma_pte *pte)
 276 {
 277         pte->val |= DMA_PTE_WRITE;
 278 }
 279
 280 static inline void dma_set_pte_snp(struct dma_pte *pte)
 281 {
 282         pte->val |= DMA_PTE_SNP;
 283 }
 284
 285 static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
 286 {
 287         pte->val = (pte->val & ~3) | (prot & 3);
 288 }
 289
 290 static inline u64 dma_pte_addr(struct dma_pte *pte)
 291 {
 292 #ifdef CONFIG_64BIT
 293         return pte->val & VTD_PAGE_MASK;
 294 #else
 295         /* Must have a full atomic 64-bit read */
 296         return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
 297 #endif
 298 }
 299
 300 static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
 301 {
 302         pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
 303 }
 304
 305 static inline bool dma_pte_present(struct dma_pte *pte)
 306 {
 307         return (pte->val & 3) != 0;
 308 }
 309
 310 static inline bool dma_pte_superpage(struct dma_pte *pte)
 311 {
 312         return (pte->val & (1 << 7));
 313 }
 314
 315 static inline int first_pte_in_page(struct dma_pte *pte)
 316 {
 317         return !((unsigned long)pte & ~VTD_PAGE_MASK);
 318 }
 319
 320 /*
 321  * This domain is a statically identity mapping domain.
 322  *      1. This domain creats a static 1:1 mapping to all usable memory.
 323  *      2. It maps to each iommu if successful.
 324  *      3. Each iommu mapps to this domain if successful.
 325  */
 326 static struct dmar_domain *si_domain;
 327 static int hw_pass_through = 1;
 328
 329 /* devices under the same p2p bridge are owned in one domain */
 330 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
 331
 332 /* domain represents a virtual machine, more than one devices
 333  * across iommus may be owned in one domain, e.g. kvm guest.
 334  */
 335 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 1)
 336
 337 /* si_domain contains mulitple devices */
 338 #define DOMAIN_FLAG_STATIC_IDENTITY     (1 << 2)
 339
 340 struct dmar_domain {
 341         int     id;                     /* domain id */
 342         int     nid;                    /* node id */
 343         unsigned long iommu_bmp;        /* bitmap of iommus this domain uses*/
 344
 345         struct list_head devices;       /* all devices' list */
 346         struct iova_domain iovad;       /* iova's that belong to this domain */
 347
 348         struct dma_pte  *pgd;           /* virtual address */
 349         int             gaw;            /* max guest address width */
 350
 351         /* adjusted guest address width, 0 is level 2 30-bit */
 352         int             agaw;
 353
 354         int             flags;          /* flags to find out type of domain */
 355
 356         int             iommu_coherency;/* indicate coherency of iommu access */
 357         int             iommu_snooping; /* indicate snooping control feature*/
 358         int             iommu_count;    /* reference count of iommu */
 359         int             iommu_superpage;/* Level of superpages supported:
 360                                            0 == 4KiB (no superpages), 1 == 2MiB,
 361                                            2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
 362         spinlock_t      iommu_lock;     /* protect iommu set in domain */
 363         u64             max_addr;       /* maximum mapped address */
 364 };
 365
 366 /* PCI domain-device relationship */
 367 struct device_domain_info {
 368         struct list_head link;  /* link to domain siblings */
 369         struct list_head global; /* link to global list */
 370         int segment;            /* PCI domain */
 371         u8 bus;                 /* PCI bus number */
 372         u8 devfn;               /* PCI devfn number */
 373         struct pci_dev *dev; /* it's NULL for PCIe-to-PCI bridge */
 374         struct intel_iommu *iommu; /* IOMMU used by this device */
 375         struct dmar_domain *domain; /* pointer to domain */
 376 };
 377
 378 static void flush_unmaps_timeout(unsigned long data);
 379
 380 DEFINE_TIMER(unmap_timer,  flush_unmaps_timeout, 0, 0);
 381
 382 #define HIGH_WATER_MARK 250
 383 struct deferred_flush_tables {
 384         int next;
 385         struct iova *iova[HIGH_WATER_MARK];
 386         struct dmar_domain *domain[HIGH_WATER_MARK];
 387 };
 388
 389 static struct deferred_flush_tables *deferred_flush;
 390
 391 /* bitmap for indexing intel_iommus */
 392 static int g_num_of_iommus;
 393
 394 static DEFINE_SPINLOCK(async_umap_flush_lock);
 395 static LIST_HEAD(unmaps_to_do);
 396
 397 static int timer_on;
 398 static long list_size;
 399
 400 static void domain_remove_dev_info(struct dmar_domain *domain);
 401
 402 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
 403 int dmar_disabled = 0;
 404 #else
 405 int dmar_disabled = 1;
 406 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
 407
 408 int intel_iommu_enabled = 0;
 409 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
 410
 411 static int dmar_map_gfx = 1;
 412 static int dmar_forcedac;
 413 static int intel_iommu_strict;
 414 static int intel_iommu_superpage = 1;
 415
 416 int intel_iommu_gfx_mapped;
 417 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
 418
 419 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 420 static DEFINE_SPINLOCK(device_domain_lock);
 421 static LIST_HEAD(device_domain_list);
 422
 423 static struct iommu_ops intel_iommu_ops;
 424
 425 static int __init intel_iommu_setup(char *str)
 426 {
 427         if (!str)
 428                 return -EINVAL;
 429         while (*str) {
 430                 if (!strncmp(str, "on", 2)) {
 431                         dmar_disabled = 0;
 432                         printk(KERN_INFO "Intel-IOMMU: enabled\n");
 433                 } else if (!strncmp(str, "off", 3)) {
 434                         dmar_disabled = 1;
 435                         printk(KERN_INFO "Intel-IOMMU: disabled\n");
 436                 } else if (!strncmp(str, "igfx_off", 8)) {
 437                         dmar_map_gfx = 0;
 438                         printk(KERN_INFO
 439                                 "Intel-IOMMU: disable GFX device mapping\n");
 440                 } else if (!strncmp(str, "forcedac", 8)) {
 441                         printk(KERN_INFO
 442                                 "Intel-IOMMU: Forcing DAC for PCI devices\n");
 443                         dmar_forcedac = 1;
 444                 } else if (!strncmp(str, "strict", 6)) {
 445                         printk(KERN_INFO
 446                                 "Intel-IOMMU: disable batched IOTLB flush\n");
 447                         intel_iommu_strict = 1;
 448                 } else if (!strncmp(str, "sp_off", 6)) {
 449                         printk(KERN_INFO
 450                                 "Intel-IOMMU: disable supported super page\n");
 451                         intel_iommu_superpage = 0;
 452                 }
 453
 454                 str += strcspn(str, ",");
 455                 while (*str == ',')
 456                         str++;
 457         }
 458         return 0;
 459 }
 460 __setup("intel_iommu=", intel_iommu_setup);
 461
 462 static struct kmem_cache *iommu_domain_cache;
 463 static struct kmem_cache *iommu_devinfo_cache;
 464 static struct kmem_cache *iommu_iova_cache;
 465
 466 static inline void *alloc_pgtable_page(int node)
 467 {
 468         struct page *page;
 469         void *vaddr = NULL;
 470
 471         page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
 472         if (page)
 473                 vaddr = page_address(page);
 474         return vaddr;
 475 }
 476
 477 static inline void free_pgtable_page(void *vaddr)
 478 {
 479         free_page((unsigned long)vaddr);
 480 }
 481
 482 static inline void *alloc_domain_mem(void)
 483 {
 484         return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
 485 }
 486
 487 static void free_domain_mem(void *vaddr)
 488 {
 489         kmem_cache_free(iommu_domain_cache, vaddr);
 490 }
 491
 492 static inline void * alloc_devinfo_mem(void)
 493 {
 494         return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
 495 }
 496
 497 static inline void free_devinfo_mem(void *vaddr)
 498 {
 499         kmem_cache_free(iommu_devinfo_cache, vaddr);
 500 }
 501
 502 struct iova *alloc_iova_mem(void)
 503 {
 504         return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
 505 }
 506
 507 void free_iova_mem(struct iova *iova)
 508 {
 509         kmem_cache_free(iommu_iova_cache, iova);
 510 }
 511
 512
 513 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
 514 {
 515         unsigned long sagaw;
 516         int agaw = -1;
 517
 518         sagaw = cap_sagaw(iommu->cap);
 519         for (agaw = width_to_agaw(max_gaw);
 520              agaw >= 0; agaw--) {
 521                 if (test_bit(agaw, &sagaw))
 522                         break;
 523         }
 524
 525         return agaw;
 526 }
 527
 528 /*
 529  * Calculate max SAGAW for each iommu.
 530  */
 531 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
 532 {
 533         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
 534 }
 535
 536 /*
 537  * calculate agaw for each iommu.
 538  * "SAGAW" may be different across iommus, use a default agaw, and
 539  * get a supported less agaw for iommus that don't support the default agaw.
 540  */
 541 int iommu_calculate_agaw(struct intel_iommu *iommu)
 542 {
 543         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 544 }
 545
 546 /* This functionin only returns single iommu in a domain */
 547 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 548 {
 549         int iommu_id;
 550
 551         /* si_domain and vm domain should not get here. */
 552         BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
 553         BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
 554
 555         iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
 556         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 557                 return NULL;
 558
 559         return g_iommus[iommu_id];
 560 }
 561
 562 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 563 {
 564         int i;
 565
 566         domain->iommu_coherency = 1;
 567
 568         for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
 569                 if (!ecap_coherent(g_iommus[i]->ecap)) {
 570                         domain->iommu_coherency = 0;
 571                         break;
 572                 }
 573         }
 574 }
 575
 576 static void domain_update_iommu_snooping(struct dmar_domain *domain)
 577 {
 578         int i;
 579
 580         domain->iommu_snooping = 1;
 581
 582         for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
 583                 if (!ecap_sc_support(g_iommus[i]->ecap)) {
 584                         domain->iommu_snooping = 0;
 585                         break;
 586                 }
 587         }
 588 }
 589
 590 static void domain_update_iommu_superpage(struct dmar_domain *domain)
 591 {
 592         struct dmar_drhd_unit *drhd;
 593         struct intel_iommu *iommu = NULL;
 594         int mask = 0xf;
 595
 596         if (!intel_iommu_superpage) {
 597                 domain->iommu_superpage = 0;
 598                 return;
 599         }
 600
 601         /* set iommu_superpage to the smallest common denominator */
 602         for_each_active_iommu(iommu, drhd) {
 603                 mask &= cap_super_page_val(iommu->cap);
 604                 if (!mask) {
 605                         break;
 606                 }
 607         }
 608         domain->iommu_superpage = fls(mask);
 609 }
 610
 611 /* Some capabilities may be different across iommus */
 612 static void domain_update_iommu_cap(struct dmar_domain *domain)
 613 {
 614         domain_update_iommu_coherency(domain);
 615         domain_update_iommu_snooping(domain);
 616         domain_update_iommu_superpage(domain);
 617 }
 618
 619 static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
 620 {
 621         struct dmar_drhd_unit *drhd = NULL;
 622         int i;
 623
 624         for_each_drhd_unit(drhd) {
 625                 if (drhd->ignored)
 626                         continue;
 627                 if (segment != drhd->segment)
 628                         continue;
 629
 630                 for (i = 0; i < drhd->devices_cnt; i++) {
 631                         if (drhd->devices[i] &&
 632                             drhd->devices[i]->bus->number == bus &&
 633                             drhd->devices[i]->devfn == devfn)
 634                                 return drhd->iommu;
 635                         if (drhd->devices[i] &&
 636                             drhd->devices[i]->subordinate &&
 637                             drhd->devices[i]->subordinate->number <= bus &&
 638                             drhd->devices[i]->subordinate->subordinate >= bus)
 639                                 return drhd->iommu;
 640                 }
 641
 642                 if (drhd->include_all)
 643                         return drhd->iommu;
 644         }
 645
 646         return NULL;
 647 }
 648
 649 static void domain_flush_cache(struct dmar_domain *domain,
 650                                void *addr, int size)
 651 {
 652         if (!domain->iommu_coherency)
 653                 clflush_cache_range(addr, size);
 654 }
 655
 656 /* Gets context entry for a given bus and devfn */
 657 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
 658                 u8 bus, u8 devfn)
 659 {
 660         struct root_entry *root;
 661         struct context_entry *context;
 662         unsigned long phy_addr;
 663         unsigned long flags;
 664
 665         spin_lock_irqsave(&iommu->lock, flags);
 666         root = &iommu->root_entry[bus];
 667         context = get_context_addr_from_root(root);
 668         if (!context) {
 669                 context = (struct context_entry *)
 670                                 alloc_pgtable_page(iommu->node);
 671                 if (!context) {
 672                         spin_unlock_irqrestore(&iommu->lock, flags);
 673                         return NULL;
 674                 }
 675                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 676                 phy_addr = virt_to_phys((void *)context);
 677                 set_root_value(root, phy_addr);
 678                 set_root_present(root);
 679                 __iommu_flush_cache(iommu, root, sizeof(*root));
 680         }
 681         spin_unlock_irqrestore(&iommu->lock, flags);
 682         return &context[devfn];
 683 }
 684
 685 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 686 {
 687         struct root_entry *root;
 688         struct context_entry *context;
 689         int ret;
 690         unsigned long flags;
 691
 692         spin_lock_irqsave(&iommu->lock, flags);
 693         root = &iommu->root_entry[bus];
 694         context = get_context_addr_from_root(root);
 695         if (!context) {
 696                 ret = 0;
 697                 goto out;
 698         }
 699         ret = context_present(&context[devfn]);
 700 out:
 701         spin_unlock_irqrestore(&iommu->lock, flags);
 702         return ret;
 703 }
 704
 705 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
 706 {
 707         struct root_entry *root;
 708         struct context_entry *context;
 709         unsigned long flags;
 710
 711         spin_lock_irqsave(&iommu->lock, flags);
 712         root = &iommu->root_entry[bus];
 713         context = get_context_addr_from_root(root);
 714         if (context) {
 715                 context_clear_entry(&context[devfn]);
 716                 __iommu_flush_cache(iommu, &context[devfn], \
 717                         sizeof(*context));
 718         }
 719         spin_unlock_irqrestore(&iommu->lock, flags);
 720 }
 721
 722 static void free_context_table(struct intel_iommu *iommu)
 723 {
 724         struct root_entry *root;
 725         int i;
 726         unsigned long flags;
 727         struct context_entry *context;
 728
 729         spin_lock_irqsave(&iommu->lock, flags);
 730         if (!iommu->root_entry) {
 731                 goto out;
 732         }
 733         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 734                 root = &iommu->root_entry[i];
 735                 context = get_context_addr_from_root(root);
 736                 if (context)
 737                         free_pgtable_page(context);
 738         }
 739         free_pgtable_page(iommu->root_entry);
 740         iommu->root_entry = NULL;
 741 out:
 742         spin_unlock_irqrestore(&iommu->lock, flags);
 743 }
 744
 745 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
 746                                       unsigned long pfn, int target_level)
 747 {
 748         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 749         struct dma_pte *parent, *pte = NULL;
 750         int level = agaw_to_level(domain->agaw);
 751         int offset;
 752
 753         BUG_ON(!domain->pgd);
 754         BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width);
 755         parent = domain->pgd;
 756
 757         while (level > 0) {
 758                 void *tmp_page;
 759
 760                 offset = pfn_level_offset(pfn, level);
 761                 pte = &parent[offset];
 762                 if (!target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
 763                         break;
 764                 if (level == target_level)
 765                         break;
 766
 767                 if (!dma_pte_present(pte)) {
 768                         uint64_t pteval;
 769
 770                         tmp_page = alloc_pgtable_page(domain->nid);
 771
 772                         if (!tmp_page)
 773                                 return NULL;
 774
 775                         domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
 776                         pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
 777                         if (cmpxchg64(&pte->val, 0ULL, pteval)) {
 778                                 /* Someone else set it while we were thinking; use theirs. */
 779                                 free_pgtable_page(tmp_page);
 780                         } else {
 781                                 dma_pte_addr(pte);
 782                                 domain_flush_cache(domain, pte, sizeof(*pte));
 783                         }
 784                 }
 785                 parent = phys_to_virt(dma_pte_addr(pte));
 786                 level--;
 787         }
 788
 789         return pte;
 790 }
 791
 792
 793 /* return address's pte at specific level */
 794 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
 795                                          unsigned long pfn,
 796                                          int level, int *large_page)
 797 {
 798         struct dma_pte *parent, *pte = NULL;
 799         int total = agaw_to_level(domain->agaw);
 800         int offset;
 801
 802         parent = domain->pgd;
 803         while (level <= total) {
 804                 offset = pfn_level_offset(pfn, total);
 805                 pte = &parent[offset];
 806                 if (level == total)
 807                         return pte;
 808
 809                 if (!dma_pte_present(pte)) {
 810                         *large_page = total;
 811                         break;
 812                 }
 813
 814                 if (pte->val & DMA_PTE_LARGE_PAGE) {
 815                         *large_page = total;
 816                         return pte;
 817                 }
 818
 819                 parent = phys_to_virt(dma_pte_addr(pte));
 820                 total--;
 821         }
 822         return NULL;
 823 }
 824
 825 /* clear last level pte, a tlb flush should be followed */
 826 static int dma_pte_clear_range(struct dmar_domain *domain,
 827                                 unsigned long start_pfn,
 828                                 unsigned long last_pfn)
 829 {
 830         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 831         unsigned int large_page = 1;
 832         struct dma_pte *first_pte, *pte;
 833         int order;
 834
 835         BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
 836         BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
 837         BUG_ON(start_pfn > last_pfn);
 838
 839         /* we don't need lock here; nobody else touches the iova range */
 840         do {
 841                 large_page = 1;
 842                 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
 843                 if (!pte) {
 844                         start_pfn = align_to_level(start_pfn + 1, large_page + 1);
 845                         continue;
 846                 }
 847                 do {
 848                         dma_clear_pte(pte);
 849                         start_pfn += lvl_to_nr_pages(large_page);
 850                         pte++;
 851                 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
 852
 853                 domain_flush_cache(domain, first_pte,
 854                                    (void *)pte - (void *)first_pte);
 855
 856         } while (start_pfn && start_pfn <= last_pfn);
 857
 858         order = (large_page - 1) * 9;
 859         return order;
 860 }
 861
 862 /* free page table pages. last level pte should already be cleared */
 863 static void dma_pte_free_pagetable(struct dmar_domain *domain,
 864                                    unsigned long start_pfn,
 865                                    unsigned long last_pfn)
 866 {
 867         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 868         struct dma_pte *first_pte, *pte;
 869         int total = agaw_to_level(domain->agaw);
 870         int level;
 871         unsigned long tmp;
 872         int large_page = 2;
 873
 874         BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
 875         BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
 876         BUG_ON(start_pfn > last_pfn);
 877
 878         /* We don't need lock here; nobody else touches the iova range */
 879         level = 2;
 880         while (level <= total) {
 881                 tmp = align_to_level(start_pfn, level);
 882
 883                 /* If we can't even clear one PTE at this level, we're done */
 884                 if (tmp + level_size(level) - 1 > last_pfn)
 885                         return;
 886
 887                 do {
 888                         large_page = level;
 889                         first_pte = pte = dma_pfn_level_pte(domain, tmp, level, &large_page);
 890                         if (large_page > level)
 891                                 level = large_page + 1;
 892                         if (!pte) {
 893                                 tmp = align_to_level(tmp + 1, level + 1);
 894                                 continue;
 895                         }
 896                         do {
 897                                 if (dma_pte_present(pte)) {
 898                                         free_pgtable_page(phys_to_virt(dma_pte_addr(pte)));
 899                                         dma_clear_pte(pte);
 900                                 }
 901                                 pte++;
 902                                 tmp += level_size(level);
 903                         } while (!first_pte_in_page(pte) &&
 904                                  tmp + level_size(level) - 1 <= last_pfn);
 905
 906                         domain_flush_cache(domain, first_pte,
 907                                            (void *)pte - (void *)first_pte);
 908
 909                 } while (tmp && tmp + level_size(level) - 1 <= last_pfn);
 910                 level++;
 911         }
 912         /* free pgd */
 913         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
 914                 free_pgtable_page(domain->pgd);
 915                 domain->pgd = NULL;
 916         }
 917 }
 918
 919 /* iommu handling */
 920 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
 921 {
 922         struct root_entry *root;
 923         unsigned long flags;
 924
 925         root = (struct root_entry *)alloc_pgtable_page(iommu->node);
 926         if (!root)
 927                 return -ENOMEM;
 928
 929         __iommu_flush_cache(iommu, root, ROOT_SIZE);
 930
 931         spin_lock_irqsave(&iommu->lock, flags);
 932         iommu->root_entry = root;
 933         spin_unlock_irqrestore(&iommu->lock, flags);
 934
 935         return 0;
 936 }
 937
 938 static void iommu_set_root_entry(struct intel_iommu *iommu)
 939 {
 940         void *addr;
 941         u32 sts;
 942         unsigned long flag;
 943
 944         addr = iommu->root_entry;
 945
 946         raw_spin_lock_irqsave(&iommu->register_lock, flag);
 947         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
 948
 949         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
 950
 951         /* Make sure hardware complete it */
 952         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 953                       readl, (sts & DMA_GSTS_RTPS), sts);
 954
 955         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
 956 }
 957
 958 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
 959 {
 960         u32 val;
 961         unsigned long flag;
 962
 963         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
 964                 return;
 965
 966         raw_spin_lock_irqsave(&iommu->register_lock, flag);
 967         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
 968
 969         /* Make sure hardware complete it */
 970         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 971                       readl, (!(val & DMA_GSTS_WBFS)), val);
 972
 973         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
 974 }
 975
 976 /* return value determine if we need a write buffer flush */
 977 static void __iommu_flush_context(struct intel_iommu *iommu,
 978                                   u16 did, u16 source_id, u8 function_mask,
 979                                   u64 type)
 980 {
 981         u64 val = 0;
 982         unsigned long flag;
 983
 984         switch (type) {
 985         case DMA_CCMD_GLOBAL_INVL:
 986                 val = DMA_CCMD_GLOBAL_INVL;
 987                 break;
 988         case DMA_CCMD_DOMAIN_INVL:
 989                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
 990                 break;
 991         case DMA_CCMD_DEVICE_INVL:
 992                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
 993                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
 994                 break;
 995         default:
 996                 BUG();
 997         }
 998         val |= DMA_CCMD_ICC;
 999
1000         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1001         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1002
1003         /* Make sure hardware complete it */
1004         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1005                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1006
1007         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1008 }
1009
1010 /* return value determine if we need a write buffer flush */
1011 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1012                                 u64 addr, unsigned int size_order, u64 type)
1013 {
1014         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1015         u64 val = 0, val_iva = 0;
1016         unsigned long flag;
1017
1018         switch (type) {
1019         case DMA_TLB_GLOBAL_FLUSH:
1020                 /* global flush doesn't need set IVA_REG */
1021                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1022                 break;
1023         case DMA_TLB_DSI_FLUSH:
1024                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1025                 break;
1026         case DMA_TLB_PSI_FLUSH:
1027                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1028                 /* Note: always flush non-leaf currently */
1029                 val_iva = size_order | addr;
1030                 break;
1031         default:
1032                 BUG();
1033         }
1034         /* Note: set drain read/write */
1035 #if 0
1036         /*
1037          * This is probably to be super secure.. Looks like we can
1038          * ignore it without any impact.
1039          */
1040         if (cap_read_drain(iommu->cap))
1041                 val |= DMA_TLB_READ_DRAIN;
1042 #endif
1043         if (cap_write_drain(iommu->cap))
1044                 val |= DMA_TLB_WRITE_DRAIN;
1045
1046         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1047         /* Note: Only uses first TLB reg currently */
1048         if (val_iva)
1049                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1050         dmar_writeq(iommu->reg + tlb_offset + 8, val);
1051
1052         /* Make sure hardware complete it */
1053         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1054                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1055
1056         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1057
1058         /* check IOTLB invalidation granularity */
1059         if (DMA_TLB_IAIG(val) == 0)
1060                 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1061         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1062                 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
1063                         (unsigned long long)DMA_TLB_IIRG(type),
1064                         (unsigned long long)DMA_TLB_IAIG(val));
1065 }
1066
1067 static struct device_domain_info *iommu_support_dev_iotlb(
1068         struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
1069 {
1070         int found = 0;
1071         unsigned long flags;
1072         struct device_domain_info *info;
1073         struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
1074
1075         if (!ecap_dev_iotlb_support(iommu->ecap))
1076                 return NULL;
1077
1078         if (!iommu->qi)
1079                 return NULL;
1080
1081         spin_lock_irqsave(&device_domain_lock, flags);
1082         list_for_each_entry(info, &domain->devices, link)
1083                 if (info->bus == bus && info->devfn == devfn) {
1084                         found = 1;
1085                         break;
1086                 }
1087         spin_unlock_irqrestore(&device_domain_lock, flags);
1088
1089         if (!found || !info->dev)
1090                 return NULL;
1091
1092         if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1093                 return NULL;
1094
1095         if (!dmar_find_matched_atsr_unit(info->dev))
1096                 return NULL;
1097
1098         info->iommu = iommu;
1099
1100         return info;
1101 }
1102
1103 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1104 {
1105         if (!info)
1106                 return;
1107
1108         pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1109 }
1110
1111 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1112 {
1113         if (!info->dev || !pci_ats_enabled(info->dev))
1114                 return;
1115
1116         pci_disable_ats(info->dev);
1117 }
1118
1119 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1120                                   u64 addr, unsigned mask)
1121 {
1122         u16 sid, qdep;
1123         unsigned long flags;
1124         struct device_domain_info *info;
1125
1126         spin_lock_irqsave(&device_domain_lock, flags);
1127         list_for_each_entry(info, &domain->devices, link) {
1128                 if (!info->dev || !pci_ats_enabled(info->dev))
1129                         continue;
1130
1131                 sid = info->bus << 8 | info->devfn;
1132                 qdep = pci_ats_queue_depth(info->dev);
1133                 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1134         }
1135         spin_unlock_irqrestore(&device_domain_lock, flags);
1136 }
1137
1138 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1139                                   unsigned long pfn, unsigned int pages, int map)
1140 {
1141         unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1142         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1143
1144         BUG_ON(pages == 0);
1145
1146         /*
1147          * Fallback to domain selective flush if no PSI support or the size is
1148          * too big.
1149          * PSI requires page size to be 2 ^ x, and the base address is naturally
1150          * aligned to the size
1151          */
1152         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1153                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1154                                                 DMA_TLB_DSI_FLUSH);
1155         else
1156                 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1157                                                 DMA_TLB_PSI_FLUSH);
1158
1159         /*
1160          * In caching mode, changes of pages from non-present to present require
1161          * flush. However, device IOTLB doesn't need to be flushed in this case.
1162          */
1163         if (!cap_caching_mode(iommu->cap) || !map)
1164                 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
1165 }
1166
1167 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1168 {
1169         u32 pmen;
1170         unsigned long flags;
1171
1172         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1173         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1174         pmen &= ~DMA_PMEN_EPM;
1175         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1176
1177         /* wait for the protected region status bit to clear */
1178         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1179                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1180
1181         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1182 }
1183
1184 static int iommu_enable_translation(struct intel_iommu *iommu)
1185 {
1186         u32 sts;
1187         unsigned long flags;
1188
1189         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1190         iommu->gcmd |= DMA_GCMD_TE;
1191         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1192
1193         /* Make sure hardware complete it */
1194         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1195                       readl, (sts & DMA_GSTS_TES), sts);
1196
1197         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1198         return 0;
1199 }
1200
1201 static int iommu_disable_translation(struct intel_iommu *iommu)
1202 {
1203         u32 sts;
1204         unsigned long flag;
1205
1206         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1207         iommu->gcmd &= ~DMA_GCMD_TE;
1208         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1209
1210         /* Make sure hardware complete it */
1211         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1212                       readl, (!(sts & DMA_GSTS_TES)), sts);
1213
1214         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1215         return 0;
1216 }
1217
1218
1219 static int iommu_init_domains(struct intel_iommu *iommu)
1220 {
1221         unsigned long ndomains;
1222         unsigned long nlongs;
1223
1224         ndomains = cap_ndoms(iommu->cap);
1225         pr_debug("IOMMU %d: Number of Domains supportd <%ld>\n", iommu->seq_id,
1226                         ndomains);
1227         nlongs = BITS_TO_LONGS(ndomains);
1228
1229         spin_lock_init(&iommu->lock);
1230
1231         /* TBD: there might be 64K domains,
1232          * consider other allocation for future chip
1233          */
1234         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1235         if (!iommu->domain_ids) {
1236                 printk(KERN_ERR "Allocating domain id array failed\n");
1237                 return -ENOMEM;
1238         }
1239         iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1240                         GFP_KERNEL);
1241         if (!iommu->domains) {
1242                 printk(KERN_ERR "Allocating domain array failed\n");
1243                 return -ENOMEM;
1244         }
1245
1246         /*
1247          * if Caching mode is set, then invalid translations are tagged
1248          * with domainid 0. Hence we need to pre-allocate it.
1249          */
1250         if (cap_caching_mode(iommu->cap))
1251                 set_bit(0, iommu->domain_ids);
1252         return 0;
1253 }
1254
1255
1256 static void domain_exit(struct dmar_domain *domain);
1257 static void vm_domain_exit(struct dmar_domain *domain);
1258
1259 void free_dmar_iommu(struct intel_iommu *iommu)
1260 {
1261         struct dmar_domain *domain;
1262         int i;
1263         unsigned long flags;
1264
1265         if ((iommu->domains) && (iommu->domain_ids)) {
1266                 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
1267                         domain = iommu->domains[i];
1268                         clear_bit(i, iommu->domain_ids);
1269
1270                         spin_lock_irqsave(&domain->iommu_lock, flags);
1271                         if (--domain->iommu_count == 0) {
1272                                 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1273                                         vm_domain_exit(domain);
1274                                 else
1275                                         domain_exit(domain);
1276                         }
1277                         spin_unlock_irqrestore(&domain->iommu_lock, flags);
1278                 }
1279         }
1280
1281         if (iommu->gcmd & DMA_GCMD_TE)
1282                 iommu_disable_translation(iommu);
1283
1284         if (iommu->irq) {
1285                 irq_set_handler_data(iommu->irq, NULL);
1286                 /* This will mask the irq */
1287                 free_irq(iommu->irq, iommu);
1288                 destroy_irq(iommu->irq);
1289         }
1290
1291         kfree(iommu->domains);
1292         kfree(iommu->domain_ids);
1293
1294         g_iommus[iommu->seq_id] = NULL;
1295
1296         /* if all iommus are freed, free g_iommus */
1297         for (i = 0; i < g_num_of_iommus; i++) {
1298                 if (g_iommus[i])
1299                         break;
1300         }
1301
1302         if (i == g_num_of_iommus)
1303                 kfree(g_iommus);
1304
1305         /* free context mapping */
1306         free_context_table(iommu);
1307 }
1308
1309 static struct dmar_domain *alloc_domain(void)
1310 {
1311         struct dmar_domain *domain;
1312
1313         domain = alloc_domain_mem();
1314         if (!domain)
1315                 return NULL;
1316
1317         domain->nid = -1;
1318         memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
1319         domain->flags = 0;
1320
1321         return domain;
1322 }
1323
1324 static int iommu_attach_domain(struct dmar_domain *domain,
1325                                struct intel_iommu *iommu)
1326 {
1327         int num;
1328         unsigned long ndomains;
1329         unsigned long flags;
1330
1331         ndomains = cap_ndoms(iommu->cap);
1332
1333         spin_lock_irqsave(&iommu->lock, flags);
1334
1335         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1336         if (num >= ndomains) {
1337                 spin_unlock_irqrestore(&iommu->lock, flags);
1338                 printk(KERN_ERR "IOMMU: no free domain ids\n");
1339                 return -ENOMEM;
1340         }
1341
1342         domain->id = num;
1343         set_bit(num, iommu->domain_ids);
1344         set_bit(iommu->seq_id, &domain->iommu_bmp);
1345         iommu->domains[num] = domain;
1346         spin_unlock_irqrestore(&iommu->lock, flags);
1347
1348         return 0;
1349 }
1350
1351 static void iommu_detach_domain(struct dmar_domain *domain,
1352                                 struct intel_iommu *iommu)
1353 {
1354         unsigned long flags;
1355         int num, ndomains;
1356         int found = 0;
1357
1358         spin_lock_irqsave(&iommu->lock, flags);
1359         ndomains = cap_ndoms(iommu->cap);
1360         for_each_set_bit(num, iommu->domain_ids, ndomains) {
1361                 if (iommu->domains[num] == domain) {
1362                         found = 1;
1363                         break;
1364                 }
1365         }
1366
1367         if (found) {
1368                 clear_bit(num, iommu->domain_ids);
1369                 clear_bit(iommu->seq_id, &domain->iommu_bmp);
1370                 iommu->domains[num] = NULL;
1371         }
1372         spin_unlock_irqrestore(&iommu->lock, flags);
1373 }
1374
1375 static struct iova_domain reserved_iova_list;
1376 static struct lock_class_key reserved_rbtree_key;
1377
1378 static int dmar_init_reserved_ranges(void)
1379 {
1380         struct pci_dev *pdev = NULL;
1381         struct iova *iova;
1382         int i;
1383
1384         init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1385
1386         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1387                 &reserved_rbtree_key);
1388
1389         /* IOAPIC ranges shouldn't be accessed by DMA */
1390         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1391                 IOVA_PFN(IOAPIC_RANGE_END));
1392         if (!iova) {
1393                 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1394                 return -ENODEV;
1395         }
1396
1397         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1398         for_each_pci_dev(pdev) {
1399                 struct resource *r;
1400
1401                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1402                         r = &pdev->resource[i];
1403                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1404                                 continue;
1405                         iova = reserve_iova(&reserved_iova_list,
1406                                             IOVA_PFN(r->start),
1407                                             IOVA_PFN(r->end));
1408                         if (!iova) {
1409                                 printk(KERN_ERR "Reserve iova failed\n");
1410                                 return -ENODEV;
1411                         }
1412                 }
1413         }
1414         return 0;
1415 }
1416
1417 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1418 {
1419         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1420 }
1421
1422 static inline int guestwidth_to_adjustwidth(int gaw)
1423 {
1424         int agaw;
1425         int r = (gaw - 12) % 9;
1426
1427         if (r == 0)
1428                 agaw = gaw;
1429         else
1430                 agaw = gaw + 9 - r;
1431         if (agaw > 64)
1432                 agaw = 64;
1433         return agaw;
1434 }
1435
1436 static int domain_init(struct dmar_domain *domain, int guest_width)
1437 {
1438         struct intel_iommu *iommu;
1439         int adjust_width, agaw;
1440         unsigned long sagaw;
1441
1442         init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1443         spin_lock_init(&domain->iommu_lock);
1444
1445         domain_reserve_special_ranges(domain);
1446
1447         /* calculate AGAW */
1448         iommu = domain_get_iommu(domain);
1449         if (guest_width > cap_mgaw(iommu->cap))
1450                 guest_width = cap_mgaw(iommu->cap);
1451         domain->gaw = guest_width;
1452         adjust_width = guestwidth_to_adjustwidth(guest_width);
1453         agaw = width_to_agaw(adjust_width);
1454         sagaw = cap_sagaw(iommu->cap);
1455         if (!test_bit(agaw, &sagaw)) {
1456                 /* hardware doesn't support it, choose a bigger one */
1457                 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1458                 agaw = find_next_bit(&sagaw, 5, agaw);
1459                 if (agaw >= 5)
1460                         return -ENODEV;
1461         }
1462         domain->agaw = agaw;
1463         INIT_LIST_HEAD(&domain->devices);
1464
1465         if (ecap_coherent(iommu->ecap))
1466                 domain->iommu_coherency = 1;
1467         else
1468                 domain->iommu_coherency = 0;
1469
1470         if (ecap_sc_support(iommu->ecap))
1471                 domain->iommu_snooping = 1;
1472         else
1473                 domain->iommu_snooping = 0;
1474
1475         domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1476         domain->iommu_count = 1;
1477         domain->nid = iommu->node;
1478
1479         /* always allocate the top pgd */
1480         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1481         if (!domain->pgd)
1482                 return -ENOMEM;
1483         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1484         return 0;
1485 }
1486
1487 static void domain_exit(struct dmar_domain *domain)
1488 {
1489         struct dmar_drhd_unit *drhd;
1490         struct intel_iommu *iommu;
1491
1492         /* Domain 0 is reserved, so dont process it */
1493         if (!domain)
1494                 return;
1495
1496         /* Flush any lazy unmaps that may reference this domain */
1497         if (!intel_iommu_strict)
1498                 flush_unmaps_timeout(0);
1499
1500         domain_remove_dev_info(domain);
1501         /* destroy iovas */
1502         put_iova_domain(&domain->iovad);
1503
1504         /* clear ptes */
1505         dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1506
1507         /* free page tables */
1508         dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1509
1510         for_each_active_iommu(iommu, drhd)
1511                 if (test_bit(iommu->seq_id, &domain->iommu_bmp))
1512                         iommu_detach_domain(domain, iommu);
1513
1514         free_domain_mem(domain);
1515 }
1516
1517 static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1518                                  u8 bus, u8 devfn, int translation)
1519 {
1520         struct context_entry *context;
1521         unsigned long flags;
1522         struct intel_iommu *iommu;
1523         struct dma_pte *pgd;
1524         unsigned long num;
1525         unsigned long ndomains;
1526         int id;
1527         int agaw;
1528         struct device_domain_info *info = NULL;
1529
1530         pr_debug("Set context mapping for %02x:%02x.%d\n",
1531                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1532
1533         BUG_ON(!domain->pgd);
1534         BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1535                translation != CONTEXT_TT_MULTI_LEVEL);
1536
1537         iommu = device_to_iommu(segment, bus, devfn);
1538         if (!iommu)
1539                 return -ENODEV;
1540
1541         context = device_to_context_entry(iommu, bus, devfn);
1542         if (!context)
1543                 return -ENOMEM;
1544         spin_lock_irqsave(&iommu->lock, flags);
1545         if (context_present(context)) {
1546                 spin_unlock_irqrestore(&iommu->lock, flags);
1547                 return 0;
1548         }
1549
1550         id = domain->id;
1551         pgd = domain->pgd;
1552
1553         if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1554             domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
1555                 int found = 0;
1556
1557                 /* find an available domain id for this device in iommu */
1558                 ndomains = cap_ndoms(iommu->cap);
1559                 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1560                         if (iommu->domains[num] == domain) {
1561                                 id = num;
1562                                 found = 1;
1563                                 break;
1564                         }
1565                 }
1566
1567                 if (found == 0) {
1568                         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1569                         if (num >= ndomains) {
1570                                 spin_unlock_irqrestore(&iommu->lock, flags);
1571                                 printk(KERN_ERR "IOMMU: no free domain ids\n");
1572                                 return -EFAULT;
1573                         }
1574
1575                         set_bit(num, iommu->domain_ids);
1576                         iommu->domains[num] = domain;
1577                         id = num;
1578                 }
1579
1580                 /* Skip top levels of page tables for
1581                  * iommu which has less agaw than default.
1582                  * Unnecessary for PT mode.
1583                  */
1584                 if (translation != CONTEXT_TT_PASS_THROUGH) {
1585                         for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1586                                 pgd = phys_to_virt(dma_pte_addr(pgd));
1587                                 if (!dma_pte_present(pgd)) {
1588                                         spin_unlock_irqrestore(&iommu->lock, flags);
1589                                         return -ENOMEM;
1590                                 }
1591                         }
1592                 }
1593         }
1594
1595         context_set_domain_id(context, id);
1596
1597         if (translation != CONTEXT_TT_PASS_THROUGH) {
1598                 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1599                 translation = info ? CONTEXT_TT_DEV_IOTLB :
1600                                      CONTEXT_TT_MULTI_LEVEL;
1601         }
1602         /*
1603          * In pass through mode, AW must be programmed to indicate the largest
1604          * AGAW value supported by hardware. And ASR is ignored by hardware.
1605          */
1606         if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
1607                 context_set_address_width(context, iommu->msagaw);
1608         else {
1609                 context_set_address_root(context, virt_to_phys(pgd));
1610                 context_set_address_width(context, iommu->agaw);
1611         }
1612
1613         context_set_translation_type(context, translation);
1614         context_set_fault_enable(context);
1615         context_set_present(context);
1616         domain_flush_cache(domain, context, sizeof(*context));
1617
1618         /*
1619          * It's a non-present to present mapping. If hardware doesn't cache
1620          * non-present entry we only need to flush the write-buffer. If the
1621          * _does_ cache non-present entries, then it does so in the special
1622          * domain #0, which we have to flush:
1623          */
1624         if (cap_caching_mode(iommu->cap)) {
1625                 iommu->flush.flush_context(iommu, 0,
1626                                            (((u16)bus) << 8) | devfn,
1627                                            DMA_CCMD_MASK_NOBIT,
1628                                            DMA_CCMD_DEVICE_INVL);
1629                 iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
1630         } else {
1631                 iommu_flush_write_buffer(iommu);
1632         }
1633         iommu_enable_dev_iotlb(info);
1634         spin_unlock_irqrestore(&iommu->lock, flags);
1635
1636         spin_lock_irqsave(&domain->iommu_lock, flags);
1637         if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {
1638                 domain->iommu_count++;
1639                 if (domain->iommu_count == 1)
1640                         domain->nid = iommu->node;
1641                 domain_update_iommu_cap(domain);
1642         }
1643         spin_unlock_irqrestore(&domain->iommu_lock, flags);
1644         return 0;
1645 }
1646
1647 static int
1648 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1649                         int translation)
1650 {
1651         int ret;
1652         struct pci_dev *tmp, *parent;
1653
1654         ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
1655                                          pdev->bus->number, pdev->devfn,
1656                                          translation);
1657         if (ret)
1658                 return ret;
1659
1660         /* dependent device mapping */
1661         tmp = pci_find_upstream_pcie_bridge(pdev);
1662         if (!tmp)
1663                 return 0;
1664         /* Secondary interface's bus number and devfn 0 */
1665         parent = pdev->bus->self;
1666         while (parent != tmp) {
1667                 ret = domain_context_mapping_one(domain,
1668                                                  pci_domain_nr(parent->bus),
1669                                                  parent->bus->number,
1670                                                  parent->devfn, translation);
1671                 if (ret)
1672                         return ret;
1673                 parent = parent->bus->self;
1674         }
1675         if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
1676                 return domain_context_mapping_one(domain,
1677                                         pci_domain_nr(tmp->subordinate),
1678                                         tmp->subordinate->number, 0,
1679                                         translation);
1680         else /* this is a legacy PCI bridge */
1681                 return domain_context_mapping_one(domain,
1682                                                   pci_domain_nr(tmp->bus),
1683                                                   tmp->bus->number,
1684                                                   tmp->devfn,
1685                                                   translation);
1686 }
1687
1688 static int domain_context_mapped(struct pci_dev *pdev)
1689 {
1690         int ret;
1691         struct pci_dev *tmp, *parent;
1692         struct intel_iommu *iommu;
1693
1694         iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1695                                 pdev->devfn);
1696         if (!iommu)
1697                 return -ENODEV;
1698
1699         ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
1700         if (!ret)
1701                 return ret;
1702         /* dependent device mapping */
1703         tmp = pci_find_upstream_pcie_bridge(pdev);
1704         if (!tmp)
1705                 return ret;
1706         /* Secondary interface's bus number and devfn 0 */
1707         parent = pdev->bus->self;
1708         while (parent != tmp) {
1709                 ret = device_context_mapped(iommu, parent->bus->number,
1710                                             parent->devfn);
1711                 if (!ret)
1712                         return ret;
1713                 parent = parent->bus->self;
1714         }
1715         if (pci_is_pcie(tmp))
1716                 return device_context_mapped(iommu, tmp->subordinate->number,
1717                                              0);
1718         else
1719                 return device_context_mapped(iommu, tmp->bus->number,
1720                                              tmp->devfn);
1721 }
1722
1723 /* Returns a number of VTD pages, but aligned to MM page size */
1724 static inline unsigned long aligned_nrpages(unsigned long host_addr,
1725                                             size_t size)
1726 {
1727         host_addr &= ~PAGE_MASK;
1728         return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1729 }
1730
1731 /* Return largest possible superpage level for a given mapping */
1732 static inline int hardware_largepage_caps(struct dmar_domain *domain,
1733                                           unsigned long iov_pfn,
1734                                           unsigned long phy_pfn,
1735                                           unsigned long pages)
1736 {
1737         int support, level = 1;
1738         unsigned long pfnmerge;
1739
1740         support = domain->iommu_superpage;
1741
1742         /* To use a large page, the virtual *and* physical addresses
1743            must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1744            of them will mean we have to use smaller pages. So just
1745            merge them and check both at once. */
1746         pfnmerge = iov_pfn | phy_pfn;
1747
1748         while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1749                 pages >>= VTD_STRIDE_SHIFT;
1750                 if (!pages)
1751                         break;
1752                 pfnmerge >>= VTD_STRIDE_SHIFT;
1753                 level++;
1754                 support--;
1755         }
1756         return level;
1757 }
1758
1759 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1760                             struct scatterlist *sg, unsigned long phys_pfn,
1761                             unsigned long nr_pages, int prot)
1762 {
1763         struct dma_pte *first_pte = NULL, *pte = NULL;
1764         phys_addr_t uninitialized_var(pteval);
1765         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1766         unsigned long sg_res;
1767         unsigned int largepage_lvl = 0;
1768         unsigned long lvl_pages = 0;
1769
1770         BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1771
1772         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1773                 return -EINVAL;
1774
1775         prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1776
1777         if (sg)
1778                 sg_res = 0;
1779         else {
1780                 sg_res = nr_pages + 1;
1781                 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1782         }
1783
1784         while (nr_pages > 0) {
1785                 uint64_t tmp;
1786
1787                 if (!sg_res) {
1788                         sg_res = aligned_nrpages(sg->offset, sg->length);
1789                         sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1790                         sg->dma_length = sg->length;
1791                         pteval = page_to_phys(sg_page(sg)) | prot;
1792                         phys_pfn = pteval >> VTD_PAGE_SHIFT;
1793                 }
1794
1795                 if (!pte) {
1796                         largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
1797
1798                         first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, largepage_lvl);
1799                         if (!pte)
1800                                 return -ENOMEM;
1801                         /* It is large page*/
1802                         if (largepage_lvl > 1)
1803                                 pteval |= DMA_PTE_LARGE_PAGE;
1804                         else
1805                                 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
1806
1807                 }
1808                 /* We don't need lock here, nobody else
1809                  * touches the iova range
1810                  */
1811                 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
1812                 if (tmp) {
1813                         static int dumps = 5;
1814                         printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1815                                iov_pfn, tmp, (unsigned long long)pteval);
1816                         if (dumps) {
1817                                 dumps--;
1818                                 debug_dma_dump_mappings(NULL);
1819                         }
1820                         WARN_ON(1);
1821                 }
1822
1823                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
1824
1825                 BUG_ON(nr_pages < lvl_pages);
1826                 BUG_ON(sg_res < lvl_pages);
1827
1828                 nr_pages -= lvl_pages;
1829                 iov_pfn += lvl_pages;
1830                 phys_pfn += lvl_pages;
1831                 pteval += lvl_pages * VTD_PAGE_SIZE;
1832                 sg_res -= lvl_pages;
1833
1834                 /* If the next PTE would be the first in a new page, then we
1835                    need to flush the cache on the entries we've just written.
1836                    And then we'll need to recalculate 'pte', so clear it and
1837                    let it get set again in the if (!pte) block above.
1838
1839                    If we're done (!nr_pages) we need to flush the cache too.
1840
1841                    Also if we've been setting superpages, we may need to
1842                    recalculate 'pte' and switch back to smaller pages for the
1843                    end of the mapping, if the trailing size is not enough to
1844                    use another superpage (i.e. sg_res < lvl_pages). */
1845                 pte++;
1846                 if (!nr_pages || first_pte_in_page(pte) ||
1847                     (largepage_lvl > 1 && sg_res < lvl_pages)) {
1848                         domain_flush_cache(domain, first_pte,
1849                                            (void *)pte - (void *)first_pte);
1850                         pte = NULL;
1851                 }
1852
1853                 if (!sg_res && nr_pages)
1854                         sg = sg_next(sg);
1855         }
1856         return 0;
1857 }
1858
1859 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1860                                     struct scatterlist *sg, unsigned long nr_pages,
1861                                     int prot)
1862 {
1863         return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
1864 }
1865
1866 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1867                                      unsigned long phys_pfn, unsigned long nr_pages,
1868                                      int prot)
1869 {
1870         return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
1871 }
1872
1873 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
1874 {
1875         if (!iommu)
1876                 return;
1877
1878         clear_context_table(iommu, bus, devfn);
1879         iommu->flush.flush_context(iommu, 0, 0, 0,
1880                                            DMA_CCMD_GLOBAL_INVL);
1881         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
1882 }
1883
1884 static void domain_remove_dev_info(struct dmar_domain *domain)
1885 {
1886         struct device_domain_info *info;
1887         unsigned long flags;
1888         struct intel_iommu *iommu;
1889
1890         spin_lock_irqsave(&device_domain_lock, flags);
1891         while (!list_empty(&domain->devices)) {
1892                 info = list_entry(domain->devices.next,
1893                         struct device_domain_info, link);
1894                 list_del(&info->link);
1895                 list_del(&info->global);
1896                 if (info->dev)
1897                         info->dev->dev.archdata.iommu = NULL;
1898                 spin_unlock_irqrestore(&device_domain_lock, flags);
1899
1900                 iommu_disable_dev_iotlb(info);
1901                 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
1902                 iommu_detach_dev(iommu, info->bus, info->devfn);
1903                 free_devinfo_mem(info);
1904
1905                 spin_lock_irqsave(&device_domain_lock, flags);
1906         }
1907         spin_unlock_irqrestore(&device_domain_lock, flags);
1908 }
1909
1910 /*
1911  * find_domain
1912  * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1913  */
1914 static struct dmar_domain *
1915 find_domain(struct pci_dev *pdev)
1916 {
1917         struct device_domain_info *info;
1918
1919         /* No lock here, assumes no domain exit in normal case */
1920         info = pdev->dev.archdata.iommu;
1921         if (info)
1922                 return info->domain;
1923         return NULL;
1924 }
1925
1926 /* domain is initialized */
1927 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1928 {
1929         struct dmar_domain *domain, *found = NULL;
1930         struct intel_iommu *iommu;
1931         struct dmar_drhd_unit *drhd;
1932         struct device_domain_info *info, *tmp;
1933         struct pci_dev *dev_tmp;
1934         unsigned long flags;
1935         int bus = 0, devfn = 0;
1936         int segment;
1937         int ret;
1938
1939         domain = find_domain(pdev);
1940         if (domain)
1941                 return domain;
1942
1943         segment = pci_domain_nr(pdev->bus);
1944
1945         dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1946         if (dev_tmp) {
1947                 if (pci_is_pcie(dev_tmp)) {
1948                         bus = dev_tmp->subordinate->number;
1949                         devfn = 0;
1950                 } else {
1951                         bus = dev_tmp->bus->number;
1952                         devfn = dev_tmp->devfn;
1953                 }
1954                 spin_lock_irqsave(&device_domain_lock, flags);
1955                 list_for_each_entry(info, &device_domain_list, global) {
1956                         if (info->segment == segment &&
1957                             info->bus == bus && info->devfn == devfn) {
1958                                 found = info->domain;
1959                                 break;
1960                         }
1961                 }
1962                 spin_unlock_irqrestore(&device_domain_lock, flags);
1963                 /* pcie-pci bridge already has a domain, uses it */
1964                 if (found) {
1965                         domain = found;
1966                         goto found_domain;
1967                 }
1968         }
1969
1970         domain = alloc_domain();
1971         if (!domain)
1972                 goto error;
1973
1974         /* Allocate new domain for the device */
1975         drhd = dmar_find_matched_drhd_unit(pdev);
1976         if (!drhd) {
1977                 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1978                         pci_name(pdev));
1979                 return NULL;
1980         }
1981         iommu = drhd->iommu;
1982
1983         ret = iommu_attach_domain(domain, iommu);
1984         if (ret) {
1985                 free_domain_mem(domain);
1986                 goto error;
1987         }
1988
1989         if (domain_init(domain, gaw)) {
1990                 domain_exit(domain);
1991                 goto error;
1992         }
1993
1994         /* register pcie-to-pci device */
1995         if (dev_tmp) {
1996                 info = alloc_devinfo_mem();
1997                 if (!info) {
1998                         domain_exit(domain);
1999                         goto error;
2000                 }
2001                 info->segment = segment;
2002                 info->bus = bus;
2003                 info->devfn = devfn;
2004                 info->dev = NULL;
2005                 info->domain = domain;
2006                 /* This domain is shared by devices under p2p bridge */
2007                 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
2008
2009                 /* pcie-to-pci bridge already has a domain, uses it */
2010                 found = NULL;
2011                 spin_lock_irqsave(&device_domain_lock, flags);
2012                 list_for_each_entry(tmp, &device_domain_list, global) {
2013                         if (tmp->segment == segment &&
2014                             tmp->bus == bus && tmp->devfn == devfn) {
2015                                 found = tmp->domain;
2016                                 break;
2017                         }
2018                 }
2019                 if (found) {
2020                         spin_unlock_irqrestore(&device_domain_lock, flags);
2021                         free_devinfo_mem(info);
2022                         domain_exit(domain);
2023                         domain = found;
2024                 } else {
2025                         list_add(&info->link, &domain->devices);
2026                         list_add(&info->global, &device_domain_list);
2027                         spin_unlock_irqrestore(&device_domain_lock, flags);
2028                 }
2029         }
2030
2031 found_domain:
2032         info = alloc_devinfo_mem();
2033         if (!info)
2034                 goto error;
2035         info->segment = segment;
2036         info->bus = pdev->bus->number;
2037         info->devfn = pdev->devfn;
2038         info->dev = pdev;
2039         info->domain = domain;
2040         spin_lock_irqsave(&device_domain_lock, flags);
2041         /* somebody is fast */
2042         found = find_domain(pdev);
2043         if (found != NULL) {
2044                 spin_unlock_irqrestore(&device_domain_lock, flags);
2045                 if (found != domain) {
2046                         domain_exit(domain);
2047                         domain = found;
2048                 }
2049                 free_devinfo_mem(info);
2050                 return domain;
2051         }
2052         list_add(&info->link, &domain->devices);
2053         list_add(&info->global, &device_domain_list);
2054         pdev->dev.archdata.iommu = info;
2055         spin_unlock_irqrestore(&device_domain_lock, flags);
2056         return domain;
2057 error:
2058         /* recheck it here, maybe others set it */
2059         return find_domain(pdev);
2060 }
2061
2062 static int iommu_identity_mapping;
2063 #define IDENTMAP_ALL            1
2064 #define IDENTMAP_GFX            2
2065 #define IDENTMAP_AZALIA         4
2066
2067 static int iommu_domain_identity_map(struct dmar_domain *domain,
2068                                      unsigned long long start,
2069                                      unsigned long long end)
2070 {
2071         unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2072         unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2073
2074         if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2075                           dma_to_mm_pfn(last_vpfn))) {
2076                 printk(KERN_ERR "IOMMU: reserve iova failed\n");
2077                 return -ENOMEM;
2078         }
2079
2080         pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2081                  start, end, domain->id);
2082         /*
2083          * RMRR range might have overlap with physical memory range,
2084          * clear it first
2085          */
2086         dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2087
2088         return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2089                                   last_vpfn - first_vpfn + 1,
2090                                   DMA_PTE_READ|DMA_PTE_WRITE);
2091 }
2092
2093 static int iommu_prepare_identity_map(struct pci_dev *pdev,
2094                                       unsigned long long start,
2095                                       unsigned long long end)
2096 {
2097         struct dmar_domain *domain;
2098         int ret;
2099
2100         domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2101         if (!domain)
2102                 return -ENOMEM;
2103
2104         /* For _hardware_ passthrough, don't bother. But for software
2105            passthrough, we do it anyway -- it may indicate a memory
2106            range which is reserved in E820, so which didn't get set
2107            up to start with in si_domain */
2108         if (domain == si_domain && hw_pass_through) {
2109                 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2110                        pci_name(pdev), start, end);
2111                 return 0;
2112         }
2113
2114         printk(KERN_INFO
2115                "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2116                pci_name(pdev), start, end);
2117
2118         if (end < start) {
2119                 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2120                         "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2121                         dmi_get_system_info(DMI_BIOS_VENDOR),
2122                         dmi_get_system_info(DMI_BIOS_VERSION),
2123                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2124                 ret = -EIO;
2125                 goto error;
2126         }
2127
2128         if (end >> agaw_to_width(domain->agaw)) {
2129                 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2130                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2131                      agaw_to_width(domain->agaw),
2132                      dmi_get_system_info(DMI_BIOS_VENDOR),
2133                      dmi_get_system_info(DMI_BIOS_VERSION),
2134                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2135                 ret = -EIO;
2136                 goto error;
2137         }
2138
2139         ret = iommu_domain_identity_map(domain, start, end);
2140         if (ret)
2141                 goto error;
2142
2143         /* context entry init */
2144         ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
2145         if (ret)
2146                 goto error;
2147
2148         return 0;
2149
2150  error:
2151         domain_exit(domain);
2152         return ret;
2153 }
2154
2155 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2156         struct pci_dev *pdev)
2157 {
2158         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2159                 return 0;
2160         return iommu_prepare_identity_map(pdev, rmrr->base_address,
2161                 rmrr->end_address);
2162 }
2163
2164 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2165 static inline void iommu_prepare_isa(void)
2166 {
2167         struct pci_dev *pdev;
2168         int ret;
2169
2170         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2171         if (!pdev)
2172                 return;
2173
2174         printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
2175         ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024 - 1);
2176
2177         if (ret)
2178                 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2179                        "floppy might not work\n");
2180
2181 }
2182 #else
2183 static inline void iommu_prepare_isa(void)
2184 {
2185         return;
2186 }
2187 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2188
2189 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2190
2191 static int __init si_domain_work_fn(unsigned long start_pfn,
2192                                     unsigned long end_pfn, void *datax)
2193 {
2194         int *ret = datax;
2195
2196         *ret = iommu_domain_identity_map(si_domain,
2197                                          (uint64_t)start_pfn << PAGE_SHIFT,
2198                                          (uint64_t)end_pfn << PAGE_SHIFT);
2199         return *ret;
2200
2201 }
2202
2203 static int __init si_domain_init(int hw)
2204 {
2205         struct dmar_drhd_unit *drhd;
2206         struct intel_iommu *iommu;
2207         int nid, ret = 0;
2208
2209         si_domain = alloc_domain();
2210         if (!si_domain)
2211                 return -EFAULT;
2212
2213         pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
2214
2215         for_each_active_iommu(iommu, drhd) {
2216                 ret = iommu_attach_domain(si_domain, iommu);
2217                 if (ret) {
2218                         domain_exit(si_domain);
2219                         return -EFAULT;
2220                 }
2221         }
2222
2223         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2224                 domain_exit(si_domain);
2225                 return -EFAULT;
2226         }
2227
2228         si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2229
2230         if (hw)
2231                 return 0;
2232
2233         for_each_online_node(nid) {
2234                 work_with_active_regions(nid, si_domain_work_fn, &ret);
2235                 if (ret)
2236                         return ret;
2237         }
2238
2239         return 0;
2240 }
2241
2242 static void domain_remove_one_dev_info(struct dmar_domain *domain,
2243                                           struct pci_dev *pdev);
2244 static int identity_mapping(struct pci_dev *pdev)
2245 {
2246         struct device_domain_info *info;
2247
2248         if (likely(!iommu_identity_mapping))
2249                 return 0;
2250
2251         info = pdev->dev.archdata.iommu;
2252         if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2253                 return (info->domain == si_domain);
2254
2255         return 0;
2256 }
2257
2258 static int domain_add_dev_info(struct dmar_domain *domain,
2259                                struct pci_dev *pdev,
2260                                int translation)
2261 {
2262         struct device_domain_info *info;
2263         unsigned long flags;
2264         int ret;
2265
2266         info = alloc_devinfo_mem();
2267         if (!info)
2268                 return -ENOMEM;
2269
2270         ret = domain_context_mapping(domain, pdev, translation);
2271         if (ret) {
2272                 free_devinfo_mem(info);
2273                 return ret;
2274         }
2275
2276         info->segment = pci_domain_nr(pdev->bus);
2277         info->bus = pdev->bus->number;
2278         info->devfn = pdev->devfn;
2279         info->dev = pdev;
2280         info->domain = domain;
2281
2282         spin_lock_irqsave(&device_domain_lock, flags);
2283         list_add(&info->link, &domain->devices);
2284         list_add(&info->global, &device_domain_list);
2285         pdev->dev.archdata.iommu = info;
2286         spin_unlock_irqrestore(&device_domain_lock, flags);
2287
2288         return 0;
2289 }
2290
2291 static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
2292 {
2293         if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2294                 return 1;
2295
2296         if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2297                 return 1;
2298
2299         if (!(iommu_identity_mapping & IDENTMAP_ALL))
2300                 return 0;
2301
2302         /*
2303          * We want to start off with all devices in the 1:1 domain, and
2304          * take them out later if we find they can't access all of memory.
2305          *
2306          * However, we can't do this for PCI devices behind bridges,
2307          * because all PCI devices behind the same bridge will end up
2308          * with the same source-id on their transactions.
2309          *
2310          * Practically speaking, we can't change things around for these
2311          * devices at run-time, because we can't be sure there'll be no
2312          * DMA transactions in flight for any of their siblings.
2313          *
2314          * So PCI devices (unless they're on the root bus) as well as
2315          * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2316          * the 1:1 domain, just in _case_ one of their siblings turns out
2317          * not to be able to map all of memory.
2318          */
2319         if (!pci_is_pcie(pdev)) {
2320                 if (!pci_is_root_bus(pdev->bus))
2321                         return 0;
2322                 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2323                         return 0;
2324         } else if (pdev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
2325                 return 0;
2326
2327         /*
2328          * At boot time, we don't yet know if devices will be 64-bit capable.
2329          * Assume that they will -- if they turn out not to be, then we can
2330          * take them out of the 1:1 domain later.
2331          */
2332         if (!startup) {
2333                 /*
2334                  * If the device's dma_mask is less than the system's memory
2335                  * size then this is not a candidate for identity mapping.
2336                  */
2337                 u64 dma_mask = pdev->dma_mask;
2338
2339                 if (pdev->dev.coherent_dma_mask &&
2340                     pdev->dev.coherent_dma_mask < dma_mask)
2341                         dma_mask = pdev->dev.coherent_dma_mask;
2342
2343                 return dma_mask >= dma_get_required_mask(&pdev->dev);
2344         }
2345
2346         return 1;
2347 }
2348
2349 static int __init iommu_prepare_static_identity_mapping(int hw)
2350 {
2351         struct pci_dev *pdev = NULL;
2352         int ret;
2353
2354         ret = si_domain_init(hw);
2355         if (ret)
2356                 return -EFAULT;
2357
2358         for_each_pci_dev(pdev) {
2359                 /* Skip Host/PCI Bridge devices */
2360                 if (IS_BRIDGE_HOST_DEVICE(pdev))
2361                         continue;
2362                 if (iommu_should_identity_map(pdev, 1)) {
2363                         printk(KERN_INFO "IOMMU: %s identity mapping for device %s\n",
2364                                hw ? "hardware" : "software", pci_name(pdev));
2365
2366                         ret = domain_add_dev_info(si_domain, pdev,
2367                                                      hw ? CONTEXT_TT_PASS_THROUGH :
2368                                                      CONTEXT_TT_MULTI_LEVEL);
2369                         if (ret)
2370                                 return ret;
2371                 }
2372         }
2373
2374         return 0;
2375 }
2376
2377 static int __init init_dmars(void)
2378 {
2379         struct dmar_drhd_unit *drhd;
2380         struct dmar_rmrr_unit *rmrr;
2381         struct pci_dev *pdev;
2382         struct intel_iommu *iommu;
2383         int i, ret;
2384
2385         /*
2386          * for each drhd
2387          *    allocate root
2388          *    initialize and program root entry to not present
2389          * endfor
2390          */
2391         for_each_drhd_unit(drhd) {
2392                 g_num_of_iommus++;
2393                 /*
2394                  * lock not needed as this is only incremented in the single
2395                  * threaded kernel __init code path all other access are read
2396                  * only
2397                  */
2398         }
2399
2400         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2401                         GFP_KERNEL);
2402         if (!g_iommus) {
2403                 printk(KERN_ERR "Allocating global iommu array failed\n");
2404                 ret = -ENOMEM;
2405                 goto error;
2406         }
2407
2408         deferred_flush = kzalloc(g_num_of_iommus *
2409                 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2410         if (!deferred_flush) {
2411                 ret = -ENOMEM;
2412                 goto error;
2413         }
2414
2415         for_each_drhd_unit(drhd) {
2416                 if (drhd->ignored)
2417                         continue;
2418
2419                 iommu = drhd->iommu;
2420                 g_iommus[iommu->seq_id] = iommu;
2421
2422                 ret = iommu_init_domains(iommu);
2423                 if (ret)
2424                         goto error;
2425
2426                 /*
2427                  * TBD:
2428                  * we could share the same root & context tables
2429                  * among all IOMMU's. Need to Split it later.
2430                  */
2431                 ret = iommu_alloc_root_entry(iommu);
2432                 if (ret) {
2433                         printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2434                         goto error;
2435                 }
2436                 if (!ecap_pass_through(iommu->ecap))
2437                         hw_pass_through = 0;
2438         }
2439
2440         /*
2441          * Start from the sane iommu hardware state.
2442          */
2443         for_each_drhd_unit(drhd) {
2444                 if (drhd->ignored)
2445                         continue;
2446
2447                 iommu = drhd->iommu;
2448
2449                 /*
2450                  * If the queued invalidation is already initialized by us
2451                  * (for example, while enabling interrupt-remapping) then
2452                  * we got the things already rolling from a sane state.
2453                  */
2454                 if (iommu->qi)
2455                         continue;
2456
2457                 /*
2458                  * Clear any previous faults.
2459                  */
2460                 dmar_fault(-1, iommu);
2461                 /*
2462                  * Disable queued invalidation if supported and already enabled
2463                  * before OS handover.
2464                  */
2465                 dmar_disable_qi(iommu);
2466         }
2467
2468         for_each_drhd_unit(drhd) {
2469                 if (drhd->ignored)
2470                         continue;
2471
2472                 iommu = drhd->iommu;
2473
2474                 if (dmar_enable_qi(iommu)) {
2475                         /*
2476                          * Queued Invalidate not enabled, use Register Based
2477                          * Invalidate
2478                          */
2479                         iommu->flush.flush_context = __iommu_flush_context;
2480                         iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2481                         printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
2482                                "invalidation\n",
2483                                 iommu->seq_id,
2484                                (unsigned long long)drhd->reg_base_addr);
2485                 } else {
2486                         iommu->flush.flush_context = qi_flush_context;
2487                         iommu->flush.flush_iotlb = qi_flush_iotlb;
2488                         printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
2489                                "invalidation\n",
2490                                 iommu->seq_id,
2491                                (unsigned long long)drhd->reg_base_addr);
2492                 }
2493         }
2494
2495         if (iommu_pass_through)
2496                 iommu_identity_mapping |= IDENTMAP_ALL;
2497
2498 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
2499         iommu_identity_mapping |= IDENTMAP_GFX;
2500 #endif
2501
2502         check_tylersburg_isoch();
2503
2504         /*
2505          * If pass through is not set or not enabled, setup context entries for
2506          * identity mappings for rmrr, gfx, and isa and may fall back to static
2507          * identity mapping if iommu_identity_mapping is set.
2508          */
2509         if (iommu_identity_mapping) {
2510                 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
2511                 if (ret) {
2512                         printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2513                         goto error;
2514                 }
2515         }
2516         /*
2517          * For each rmrr
2518          *   for each dev attached to rmrr
2519          *   do
2520          *     locate drhd for dev, alloc domain for dev
2521          *     allocate free domain
2522          *     allocate page table entries for rmrr
2523          *     if context not allocated for bus
2524          *           allocate and init context
2525          *           set present in root table for this bus
2526          *     init context with domain, translation etc
2527          *    endfor
2528          * endfor
2529          */
2530         printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2531         for_each_rmrr_units(rmrr) {
2532                 for (i = 0; i < rmrr->devices_cnt; i++) {
2533                         pdev = rmrr->devices[i];
2534                         /*
2535                          * some BIOS lists non-exist devices in DMAR
2536                          * table.
2537                          */
2538                         if (!pdev)
2539                                 continue;
2540                         ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2541                         if (ret)
2542                                 printk(KERN_ERR
2543                                        "IOMMU: mapping reserved region failed\n");
2544                 }
2545         }
2546
2547         iommu_prepare_isa();
2548
2549         /*
2550          * for each drhd
2551          *   enable fault log
2552          *   global invalidate context cache
2553          *   global invalidate iotlb
2554          *   enable translation
2555          */
2556         for_each_drhd_unit(drhd) {
2557                 if (drhd->ignored) {
2558                         /*
2559                          * we always have to disable PMRs or DMA may fail on
2560                          * this device
2561                          */
2562                         if (force_on)
2563                                 iommu_disable_protect_mem_regions(drhd->iommu);
2564                         continue;
2565                 }
2566                 iommu = drhd->iommu;
2567
2568                 iommu_flush_write_buffer(iommu);
2569
2570                 ret = dmar_set_interrupt(iommu);
2571                 if (ret)
2572                         goto error;
2573
2574                 iommu_set_root_entry(iommu);
2575
2576                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
2577                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2578
2579                 ret = iommu_enable_translation(iommu);
2580                 if (ret)
2581                         goto error;
2582
2583                 iommu_disable_protect_mem_regions(iommu);
2584         }
2585
2586         return 0;
2587 error:
2588         for_each_drhd_unit(drhd) {
2589                 if (drhd->ignored)
2590                         continue;
2591                 iommu = drhd->iommu;
2592                 free_iommu(iommu);
2593         }
2594         kfree(g_iommus);
2595         return ret;
2596 }
2597
2598 /* This takes a number of _MM_ pages, not VTD pages */
2599 static struct iova *intel_alloc_iova(struct device *dev,
2600                                      struct dmar_domain *domain,
2601                                      unsigned long nrpages, uint64_t dma_mask)
2602 {
2603         struct pci_dev *pdev = to_pci_dev(dev);
2604         struct iova *iova = NULL;
2605
2606         /* Restrict dma_mask to the width that the iommu can handle */
2607         dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2608
2609         if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
2610                 /*
2611                  * First try to allocate an io virtual address in
2612                  * DMA_BIT_MASK(32) and if that fails then try allocating
2613                  * from higher range
2614                  */
2615                 iova = alloc_iova(&domain->iovad, nrpages,
2616                                   IOVA_PFN(DMA_BIT_MASK(32)), 1);
2617                 if (iova)
2618                         return iova;
2619         }
2620         iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2621         if (unlikely(!iova)) {
2622                 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2623                        nrpages, pci_name(pdev));
2624                 return NULL;
2625         }
2626
2627         return iova;
2628 }
2629
2630 static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
2631 {
2632         struct dmar_domain *domain;
2633         int ret;
2634
2635         domain = get_domain_for_dev(pdev,
2636                         DEFAULT_DOMAIN_ADDRESS_WIDTH);
2637         if (!domain) {
2638                 printk(KERN_ERR
2639                         "Allocating domain for %s failed", pci_name(pdev));
2640                 return NULL;
2641         }
2642
2643         /* make sure context mapping is ok */
2644         if (unlikely(!domain_context_mapped(pdev))) {
2645                 ret = domain_context_mapping(domain, pdev,
2646                                              CONTEXT_TT_MULTI_LEVEL);
2647                 if (ret) {
2648                         printk(KERN_ERR
2649                                 "Domain context map for %s failed",
2650                                 pci_name(pdev));
2651                         return NULL;
2652                 }
2653         }
2654
2655         return domain;
2656 }
2657
2658 static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
2659 {
2660         struct device_domain_info *info;
2661
2662         /* No lock here, assumes no domain exit in normal case */
2663         info = dev->dev.archdata.iommu;
2664         if (likely(info))
2665                 return info->domain;
2666
2667         return __get_valid_domain_for_dev(dev);
2668 }
2669
2670 static int iommu_dummy(struct pci_dev *pdev)
2671 {
2672         return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2673 }
2674
2675 /* Check if the pdev needs to go through non-identity map and unmap process.*/
2676 static int iommu_no_mapping(struct device *dev)
2677 {
2678         struct pci_dev *pdev;
2679         int found;
2680
2681         if (unlikely(dev->bus != &pci_bus_type))
2682                 return 1;
2683
2684         pdev = to_pci_dev(dev);
2685         if (iommu_dummy(pdev))
2686                 return 1;
2687
2688         if (!iommu_identity_mapping)
2689                 return 0;
2690
2691         found = identity_mapping(pdev);
2692         if (found) {
2693                 if (iommu_should_identity_map(pdev, 0))
2694                         return 1;
2695                 else {
2696                         /*
2697                          * 32 bit DMA is removed from si_domain and fall back
2698                          * to non-identity mapping.
2699                          */
2700                         domain_remove_one_dev_info(si_domain, pdev);
2701                         printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2702                                pci_name(pdev));
2703                         return 0;
2704                 }
2705         } else {
2706                 /*
2707                  * In case of a detached 64 bit DMA device from vm, the device
2708                  * is put into si_domain for identity mapping.
2709                  */
2710                 if (iommu_should_identity_map(pdev, 0)) {
2711                         int ret;
2712                         ret = domain_add_dev_info(si_domain, pdev,
2713                                                   hw_pass_through ?
2714                                                   CONTEXT_TT_PASS_THROUGH :
2715                                                   CONTEXT_TT_MULTI_LEVEL);
2716                         if (!ret) {
2717                                 printk(KERN_INFO "64bit %s uses identity mapping\n",
2718                                        pci_name(pdev));
2719                                 return 1;
2720                         }
2721                 }
2722         }
2723
2724         return 0;
2725 }
2726
2727 static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2728                                      size_t size, int dir, u64 dma_mask)
2729 {
2730         struct pci_dev *pdev = to_pci_dev(hwdev);
2731         struct dmar_domain *domain;
2732         phys_addr_t start_paddr;
2733         struct iova *iova;
2734         int prot = 0;
2735         int ret;
2736         struct intel_iommu *iommu;
2737         unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
2738
2739         BUG_ON(dir == DMA_NONE);
2740
2741         if (iommu_no_mapping(hwdev))
2742                 return paddr;
2743
2744         domain = get_valid_domain_for_dev(pdev);
2745         if (!domain)
2746                 return 0;
2747
2748         iommu = domain_get_iommu(domain);
2749         size = aligned_nrpages(paddr, size);
2750
2751         iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size), dma_mask);
2752         if (!iova)
2753                 goto error;
2754
2755         /*
2756          * Check if DMAR supports zero-length reads on write only
2757          * mappings..
2758          */
2759         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2760                         !cap_zlr(iommu->cap))
2761                 prot |= DMA_PTE_READ;
2762         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2763                 prot |= DMA_PTE_WRITE;
2764         /*
2765          * paddr - (paddr + size) might be partial page, we should map the whole
2766          * page.  Note: if two part of one page are separately mapped, we
2767          * might have two guest_addr mapping to the same host paddr, but this
2768          * is not a big problem
2769          */
2770         ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
2771                                  mm_to_dma_pfn(paddr_pfn), size, prot);
2772         if (ret)
2773                 goto error;
2774
2775         /* it's a non-present to present mapping. Only flush if caching mode */
2776         if (cap_caching_mode(iommu->cap))
2777                 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 1);
2778         else
2779                 iommu_flush_write_buffer(iommu);
2780
2781         start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2782         start_paddr += paddr & ~PAGE_MASK;
2783         return start_paddr;
2784
2785 error:
2786         if (iova)
2787                 __free_iova(&domain->iovad, iova);
2788         printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
2789                 pci_name(pdev), size, (unsigned long long)paddr, dir);
2790         return 0;
2791 }
2792
2793 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2794                                  unsigned long offset, size_t size,
2795                                  enum dma_data_direction dir,
2796                                  struct dma_attrs *attrs)
2797 {
2798         return __intel_map_single(dev, page_to_phys(page) + offset, size,
2799                                   dir, to_pci_dev(dev)->dma_mask);
2800 }
2801
2802 static void flush_unmaps(void)
2803 {
2804         int i, j;
2805
2806         timer_on = 0;
2807
2808         /* just flush them all */
2809         for (i = 0; i < g_num_of_iommus; i++) {
2810                 struct intel_iommu *iommu = g_iommus[i];
2811                 if (!iommu)
2812                         continue;
2813
2814                 if (!deferred_flush[i].next)
2815                         continue;
2816
2817                 /* In caching mode, global flushes turn emulation expensive */
2818                 if (!cap_caching_mode(iommu->cap))
2819                         iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2820                                          DMA_TLB_GLOBAL_FLUSH);
2821                 for (j = 0; j < deferred_flush[i].next; j++) {
2822                         unsigned long mask;
2823                         struct iova *iova = deferred_flush[i].iova[j];
2824                         struct dmar_domain *domain = deferred_flush[i].domain[j];
2825
2826                         /* On real hardware multiple invalidations are expensive */
2827                         if (cap_caching_mode(iommu->cap))
2828                                 iommu_flush_iotlb_psi(iommu, domain->id,
2829                                 iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 0);
2830                         else {
2831                                 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
2832                                 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
2833                                                 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
2834                         }
2835                         __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
2836                 }
2837                 deferred_flush[i].next = 0;
2838         }
2839
2840         list_size = 0;
2841 }
2842
2843 static void flush_unmaps_timeout(unsigned long data)
2844 {
2845         unsigned long flags;
2846
2847         spin_lock_irqsave(&async_umap_flush_lock, flags);
2848         flush_unmaps();
2849         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2850 }
2851
2852 static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2853 {
2854         unsigned long flags;
2855         int next, iommu_id;
2856         struct intel_iommu *iommu;
2857
2858         spin_lock_irqsave(&async_umap_flush_lock, flags);
2859         if (list_size == HIGH_WATER_MARK)
2860                 flush_unmaps();
2861
2862         iommu = domain_get_iommu(dom);
2863         iommu_id = iommu->seq_id;
2864
2865         next = deferred_flush[iommu_id].next;
2866         deferred_flush[iommu_id].domain[next] = dom;
2867         deferred_flush[iommu_id].iova[next] = iova;
2868         deferred_flush[iommu_id].next++;
2869
2870         if (!timer_on) {
2871                 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2872                 timer_on = 1;
2873         }
2874         list_size++;
2875         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2876 }
2877
2878 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2879                              size_t size, enum dma_data_direction dir,
2880                              struct dma_attrs *attrs)
2881 {
2882         struct pci_dev *pdev = to_pci_dev(dev);
2883         struct dmar_domain *domain;
2884         unsigned long start_pfn, last_pfn;
2885         struct iova *iova;
2886         struct intel_iommu *iommu;
2887
2888         if (iommu_no_mapping(dev))
2889                 return;
2890
2891         domain = find_domain(pdev);
2892         BUG_ON(!domain);
2893
2894         iommu = domain_get_iommu(domain);
2895
2896         iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
2897         if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
2898                       (unsigned long long)dev_addr))
2899                 return;
2900
2901         start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2902         last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2903
2904         pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2905                  pci_name(pdev), start_pfn, last_pfn);
2906
2907         /*  clear the whole page */
2908         dma_pte_clear_range(domain, start_pfn, last_pfn);
2909
2910         /* free page tables */
2911         dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2912
2913         if (intel_iommu_strict) {
2914                 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2915                                       last_pfn - start_pfn + 1, 0);
2916                 /* free iova */
2917                 __free_iova(&domain->iovad, iova);
2918         } else {
2919                 add_unmap(domain, iova);
2920                 /*
2921                  * queue up the release of the unmap to save the 1/6th of the
2922                  * cpu used up by the iotlb flush operation...
2923                  */
2924         }
2925 }
2926
2927 static void *intel_alloc_coherent(struct device *hwdev, size_t size,
2928                                   dma_addr_t *dma_handle, gfp_t flags)
2929 {
2930         void *vaddr;
2931         int order;
2932
2933         size = PAGE_ALIGN(size);
2934         order = get_order(size);
2935
2936         if (!iommu_no_mapping(hwdev))
2937                 flags &= ~(GFP_DMA | GFP_DMA32);
2938         else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
2939                 if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
2940                         flags |= GFP_DMA;
2941                 else
2942                         flags |= GFP_DMA32;
2943         }
2944
2945         vaddr = (void *)__get_free_pages(flags, order);
2946         if (!vaddr)
2947                 return NULL;
2948         memset(vaddr, 0, size);
2949
2950         *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2951                                          DMA_BIDIRECTIONAL,
2952                                          hwdev->coherent_dma_mask);
2953         if (*dma_handle)
2954                 return vaddr;
2955         free_pages((unsigned long)vaddr, order);
2956         return NULL;
2957 }
2958
2959 static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2960                                 dma_addr_t dma_handle)
2961 {
2962         int order;
2963
2964         size = PAGE_ALIGN(size);
2965         order = get_order(size);
2966
2967         intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
2968         free_pages((unsigned long)vaddr, order);
2969 }
2970
2971 static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2972                            int nelems, enum dma_data_direction dir,
2973                            struct dma_attrs *attrs)
2974 {
2975         struct pci_dev *pdev = to_pci_dev(hwdev);
2976         struct dmar_domain *domain;
2977         unsigned long start_pfn, last_pfn;
2978         struct iova *iova;
2979         struct intel_iommu *iommu;
2980
2981         if (iommu_no_mapping(hwdev))
2982                 return;
2983
2984         domain = find_domain(pdev);
2985         BUG_ON(!domain);
2986
2987         iommu = domain_get_iommu(domain);
2988
2989         iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
2990         if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
2991                       (unsigned long long)sglist[0].dma_address))
2992                 return;
2993
2994         start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2995         last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2996
2997         /*  clear the whole page */
2998         dma_pte_clear_range(domain, start_pfn, last_pfn);
2999
3000         /* free page tables */
3001         dma_pte_free_pagetable(domain, start_pfn, last_pfn);
3002
3003         if (intel_iommu_strict) {
3004                 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
3005                                       last_pfn - start_pfn + 1, 0);
3006                 /* free iova */
3007                 __free_iova(&domain->iovad, iova);
3008         } else {
3009                 add_unmap(domain, iova);
3010                 /*
3011                  * queue up the release of the unmap to save the 1/6th of the
3012                  * cpu used up by the iotlb flush operation...
3013                  */
3014         }
3015 }
3016
3017 static int intel_nontranslate_map_sg(struct device *hddev,
3018         struct scatterlist *sglist, int nelems, int dir)
3019 {
3020         int i;
3021         struct scatterlist *sg;
3022
3023         for_each_sg(sglist, sg, nelems, i) {
3024                 BUG_ON(!sg_page(sg));
3025                 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
3026                 sg->dma_length = sg->length;
3027         }
3028         return nelems;
3029 }
3030
3031 static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
3032                         enum dma_data_direction dir, struct dma_attrs *attrs)
3033 {
3034         int i;
3035         struct pci_dev *pdev = to_pci_dev(hwdev);
3036         struct dmar_domain *domain;
3037         size_t size = 0;
3038         int prot = 0;
3039         struct iova *iova = NULL;
3040         int ret;
3041         struct scatterlist *sg;
3042         unsigned long start_vpfn;
3043         struct intel_iommu *iommu;
3044
3045         BUG_ON(dir == DMA_NONE);
3046         if (iommu_no_mapping(hwdev))
3047                 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
3048
3049         domain = get_valid_domain_for_dev(pdev);
3050         if (!domain)
3051                 return 0;
3052
3053         iommu = domain_get_iommu(domain);
3054
3055         for_each_sg(sglist, sg, nelems, i)
3056                 size += aligned_nrpages(sg->offset, sg->length);
3057
3058         iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
3059                                 pdev->dma_mask);
3060         if (!iova) {
3061                 sglist->dma_length = 0;
3062                 return 0;
3063         }
3064
3065         /*
3066          * Check if DMAR supports zero-length reads on write only
3067          * mappings..
3068          */
3069         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3070                         !cap_zlr(iommu->cap))
3071                 prot |= DMA_PTE_READ;
3072         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3073                 prot |= DMA_PTE_WRITE;
3074
3075         start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
3076
3077         ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3078         if (unlikely(ret)) {
3079                 /*  clear the page */
3080                 dma_pte_clear_range(domain, start_vpfn,
3081                                     start_vpfn + size - 1);
3082                 /* free page tables */
3083                 dma_pte_free_pagetable(domain, start_vpfn,
3084                                        start_vpfn + size - 1);
3085                 /* free iova */
3086                 __free_iova(&domain->iovad, iova);
3087                 return 0;
3088         }
3089
3090         /* it's a non-present to present mapping. Only flush if caching mode */
3091         if (cap_caching_mode(iommu->cap))
3092                 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 1);
3093         else
3094                 iommu_flush_write_buffer(iommu);
3095
3096         return nelems;
3097 }
3098
3099 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3100 {
3101         return !dma_addr;
3102 }
3103
3104 struct dma_map_ops intel_dma_ops = {
3105         .alloc_coherent = intel_alloc_coherent,
3106         .free_coherent = intel_free_coherent,
3107         .map_sg = intel_map_sg,
3108         .unmap_sg = intel_unmap_sg,
3109         .map_page = intel_map_page,
3110         .unmap_page = intel_unmap_page,
3111         .mapping_error = intel_mapping_error,
3112 };
3113
3114 static inline int iommu_domain_cache_init(void)
3115 {
3116         int ret = 0;
3117
3118         iommu_domain_cache = kmem_cache_create("iommu_domain",
3119                                          sizeof(struct dmar_domain),
3120                                          0,
3121                                          SLAB_HWCACHE_ALIGN,
3122
3123                                          NULL);
3124         if (!iommu_domain_cache) {
3125                 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
3126                 ret = -ENOMEM;
3127         }
3128
3129         return ret;
3130 }
3131
3132 static inline int iommu_devinfo_cache_init(void)
3133 {
3134         int ret = 0;
3135
3136         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3137                                          sizeof(struct device_domain_info),
3138                                          0,
3139                                          SLAB_HWCACHE_ALIGN,
3140                                          NULL);
3141         if (!iommu_devinfo_cache) {
3142                 printk(KERN_ERR "Couldn't create devinfo cache\n");
3143                 ret = -ENOMEM;
3144         }
3145
3146         return ret;
3147 }
3148
3149 static inline int iommu_iova_cache_init(void)
3150 {
3151         int ret = 0;
3152
3153         iommu_iova_cache = kmem_cache_create("iommu_iova",
3154                                          sizeof(struct iova),
3155                                          0,
3156                                          SLAB_HWCACHE_ALIGN,
3157                                          NULL);
3158         if (!iommu_iova_cache) {
3159                 printk(KERN_ERR "Couldn't create iova cache\n");
3160                 ret = -ENOMEM;
3161         }
3162
3163         return ret;
3164 }
3165
3166 static int __init iommu_init_mempool(void)
3167 {
3168         int ret;
3169         ret = iommu_iova_cache_init();
3170         if (ret)
3171                 return ret;
3172
3173         ret = iommu_domain_cache_init();
3174         if (ret)
3175                 goto domain_error;
3176
3177         ret = iommu_devinfo_cache_init();
3178         if (!ret)
3179                 return ret;
3180
3181         kmem_cache_destroy(iommu_domain_cache);
3182 domain_error:
3183         kmem_cache_destroy(iommu_iova_cache);
3184
3185         return -ENOMEM;
3186 }
3187
3188 static void __init iommu_exit_mempool(void)
3189 {
3190         kmem_cache_destroy(iommu_devinfo_cache);
3191         kmem_cache_destroy(iommu_domain_cache);
3192         kmem_cache_destroy(iommu_iova_cache);
3193
3194 }
3195
3196 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3197 {
3198         struct dmar_drhd_unit *drhd;
3199         u32 vtbar;
3200         int rc;
3201
3202         /* We know that this device on this chipset has its own IOMMU.
3203          * If we find it under a different IOMMU, then the BIOS is lying
3204          * to us. Hope that the IOMMU for this device is actually
3205          * disabled, and it needs no translation...
3206          */
3207         rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3208         if (rc) {
3209                 /* "can't" happen */
3210                 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3211                 return;
3212         }
3213         vtbar &= 0xffff0000;
3214
3215         /* we know that the this iommu should be at offset 0xa000 from vtbar */
3216         drhd = dmar_find_matched_drhd_unit(pdev);
3217         if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3218                             TAINT_FIRMWARE_WORKAROUND,
3219                             "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3220                 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3221 }
3222 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3223
3224 static void __init init_no_remapping_devices(void)
3225 {
3226         struct dmar_drhd_unit *drhd;
3227
3228         for_each_drhd_unit(drhd) {
3229                 if (!drhd->include_all) {
3230                         int i;
3231                         for (i = 0; i < drhd->devices_cnt; i++)
3232                                 if (drhd->devices[i] != NULL)
3233                                         break;
3234                         /* ignore DMAR unit if no pci devices exist */
3235                         if (i == drhd->devices_cnt)
3236                                 drhd->ignored = 1;
3237                 }
3238         }
3239
3240         for_each_drhd_unit(drhd) {
3241                 int i;
3242                 if (drhd->ignored || drhd->include_all)
3243                         continue;
3244
3245                 for (i = 0; i < drhd->devices_cnt; i++)
3246                         if (drhd->devices[i] &&
3247                             !IS_GFX_DEVICE(drhd->devices[i]))
3248                                 break;
3249
3250                 if (i < drhd->devices_cnt)
3251                         continue;
3252
3253                 /* This IOMMU has *only* gfx devices. Either bypass it or
3254                    set the gfx_mapped flag, as appropriate */
3255                 if (dmar_map_gfx) {
3256                         intel_iommu_gfx_mapped = 1;
3257                 } else {
3258                         drhd->ignored = 1;
3259                         for (i = 0; i < drhd->devices_cnt; i++) {
3260                                 if (!drhd->devices[i])
3261                                         continue;
3262                                 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3263                         }
3264                 }
3265         }
3266 }
3267
3268 #ifdef CONFIG_SUSPEND
3269 static int init_iommu_hw(void)
3270 {
3271         struct dmar_drhd_unit *drhd;
3272         struct intel_iommu *iommu = NULL;
3273
3274         for_each_active_iommu(iommu, drhd)
3275                 if (iommu->qi)
3276                         dmar_reenable_qi(iommu);
3277
3278         for_each_iommu(iommu, drhd) {
3279                 if (drhd->ignored) {
3280                         /*
3281                          * we always have to disable PMRs or DMA may fail on
3282                          * this device
3283                          */
3284                         if (force_on)
3285                                 iommu_disable_protect_mem_regions(iommu);
3286                         continue;
3287                 }
3288
3289                 iommu_flush_write_buffer(iommu);
3290
3291                 iommu_set_root_entry(iommu);
3292
3293                 iommu->flush.flush_context(iommu, 0, 0, 0,
3294                                            DMA_CCMD_GLOBAL_INVL);
3295                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3296                                          DMA_TLB_GLOBAL_FLUSH);
3297                 if (iommu_enable_translation(iommu))
3298                         return 1;
3299                 iommu_disable_protect_mem_regions(iommu);
3300         }
3301
3302         return 0;
3303 }
3304
3305 static void iommu_flush_all(void)
3306 {
3307         struct dmar_drhd_unit *drhd;
3308         struct intel_iommu *iommu;
3309
3310         for_each_active_iommu(iommu, drhd) {
3311                 iommu->flush.flush_context(iommu, 0, 0, 0,
3312                                            DMA_CCMD_GLOBAL_INVL);
3313                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3314                                          DMA_TLB_GLOBAL_FLUSH);
3315         }
3316 }
3317
3318 static int iommu_suspend(void)
3319 {
3320         struct dmar_drhd_unit *drhd;
3321         struct intel_iommu *iommu = NULL;
3322         unsigned long flag;
3323
3324         for_each_active_iommu(iommu, drhd) {
3325                 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3326                                                  GFP_ATOMIC);
3327                 if (!iommu->iommu_state)
3328                         goto nomem;
3329         }
3330
3331         iommu_flush_all();
3332
3333         for_each_active_iommu(iommu, drhd) {
3334                 iommu_disable_translation(iommu);
3335
3336                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3337
3338                 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3339                         readl(iommu->reg + DMAR_FECTL_REG);
3340                 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3341                         readl(iommu->reg + DMAR_FEDATA_REG);
3342                 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3343                         readl(iommu->reg + DMAR_FEADDR_REG);
3344                 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3345                         readl(iommu->reg + DMAR_FEUADDR_REG);
3346
3347                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3348         }
3349         return 0;
3350
3351 nomem:
3352         for_each_active_iommu(iommu, drhd)
3353                 kfree(iommu->iommu_state);
3354
3355         return -ENOMEM;
3356 }
3357
3358 static void iommu_resume(void)
3359 {
3360         struct dmar_drhd_unit *drhd;
3361         struct intel_iommu *iommu = NULL;
3362         unsigned long flag;
3363
3364         if (init_iommu_hw()) {
3365                 if (force_on)
3366                         panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3367                 else
3368                         WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3369                 return;
3370         }
3371
3372         for_each_active_iommu(iommu, drhd) {
3373
3374                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3375
3376                 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3377                         iommu->reg + DMAR_FECTL_REG);
3378                 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3379                         iommu->reg + DMAR_FEDATA_REG);
3380                 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3381                         iommu->reg + DMAR_FEADDR_REG);
3382                 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3383                         iommu->reg + DMAR_FEUADDR_REG);
3384
3385                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3386         }
3387
3388         for_each_active_iommu(iommu, drhd)
3389                 kfree(iommu->iommu_state);
3390 }
3391
3392 static struct syscore_ops iommu_syscore_ops = {
3393         .resume         = iommu_resume,
3394         .suspend        = iommu_suspend,
3395 };
3396
3397 static void __init init_iommu_pm_ops(void)
3398 {
3399         register_syscore_ops(&iommu_syscore_ops);
3400 }
3401
3402 #else
3403 static inline void init_iommu_pm_ops(void) {}
3404 #endif  /* CONFIG_PM */
3405
3406 LIST_HEAD(dmar_rmrr_units);
3407
3408 static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
3409 {
3410         list_add(&rmrr->list, &dmar_rmrr_units);
3411 }
3412
3413
3414 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header)
3415 {
3416         struct acpi_dmar_reserved_memory *rmrr;
3417         struct dmar_rmrr_unit *rmrru;
3418
3419         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3420         if (!rmrru)
3421                 return -ENOMEM;
3422
3423         rmrru->hdr = header;
3424         rmrr = (struct acpi_dmar_reserved_memory *)header;
3425         rmrru->base_address = rmrr->base_address;
3426         rmrru->end_address = rmrr->end_address;
3427
3428         dmar_register_rmrr_unit(rmrru);
3429         return 0;
3430 }
3431
3432 static int __init
3433 rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
3434 {
3435         struct acpi_dmar_reserved_memory *rmrr;
3436         int ret;
3437
3438         rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
3439         ret = dmar_parse_dev_scope((void *)(rmrr + 1),
3440                 ((void *)rmrr) + rmrr->header.length,
3441                 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
3442
3443         if (ret || (rmrru->devices_cnt == 0)) {
3444                 list_del(&rmrru->list);
3445                 kfree(rmrru);
3446         }
3447         return ret;
3448 }
3449
3450 static LIST_HEAD(dmar_atsr_units);
3451
3452 int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
3453 {
3454         struct acpi_dmar_atsr *atsr;
3455         struct dmar_atsr_unit *atsru;
3456
3457         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3458         atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
3459         if (!atsru)
3460                 return -ENOMEM;
3461
3462         atsru->hdr = hdr;
3463         atsru->include_all = atsr->flags & 0x1;
3464
3465         list_add(&atsru->list, &dmar_atsr_units);
3466
3467         return 0;
3468 }
3469
3470 static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
3471 {
3472         int rc;
3473         struct acpi_dmar_atsr *atsr;
3474
3475         if (atsru->include_all)
3476                 return 0;
3477
3478         atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3479         rc = dmar_parse_dev_scope((void *)(atsr + 1),
3480                                 (void *)atsr + atsr->header.length,
3481                                 &atsru->devices_cnt, &atsru->devices,
3482                                 atsr->segment);
3483         if (rc || !atsru->devices_cnt) {
3484                 list_del(&atsru->list);
3485                 kfree(atsru);
3486         }
3487
3488         return rc;
3489 }
3490
3491 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
3492 {
3493         int i;
3494         struct pci_bus *bus;
3495         struct acpi_dmar_atsr *atsr;
3496         struct dmar_atsr_unit *atsru;
3497
3498         dev = pci_physfn(dev);
3499
3500         list_for_each_entry(atsru, &dmar_atsr_units, list) {
3501                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3502                 if (atsr->segment == pci_domain_nr(dev->bus))
3503                         goto found;
3504         }
3505
3506         return 0;
3507
3508 found:
3509         for (bus = dev->bus; bus; bus = bus->parent) {
3510                 struct pci_dev *bridge = bus->self;
3511
3512                 if (!bridge || !pci_is_pcie(bridge) ||
3513                     bridge->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
3514                         return 0;
3515
3516                 if (bridge->pcie_type == PCI_EXP_TYPE_ROOT_PORT) {
3517                         for (i = 0; i < atsru->devices_cnt; i++)
3518                                 if (atsru->devices[i] == bridge)
3519                                         return 1;
3520                         break;
3521                 }
3522         }
3523
3524         if (atsru->include_all)
3525                 return 1;
3526
3527         return 0;
3528 }
3529
3530 int __init dmar_parse_rmrr_atsr_dev(void)
3531 {
3532         struct dmar_rmrr_unit *rmrr, *rmrr_n;
3533         struct dmar_atsr_unit *atsr, *atsr_n;
3534         int ret = 0;
3535
3536         list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
3537                 ret = rmrr_parse_dev(rmrr);
3538                 if (ret)
3539                         return ret;
3540         }
3541
3542         list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
3543                 ret = atsr_parse_dev(atsr);
3544                 if (ret)
3545                         return ret;
3546         }
3547
3548         return ret;
3549 }
3550
3551 /*
3552  * Here we only respond to action of unbound device from driver.
3553  *
3554  * Added device is not attached to its DMAR domain here yet. That will happen
3555  * when mapping the device to iova.
3556  */
3557 static int device_notifier(struct notifier_block *nb,
3558                                   unsigned long action, void *data)
3559 {
3560         struct device *dev = data;
3561         struct pci_dev *pdev = to_pci_dev(dev);
3562         struct dmar_domain *domain;
3563
3564         if (iommu_no_mapping(dev))
3565                 return 0;
3566
3567         domain = find_domain(pdev);
3568         if (!domain)
3569                 return 0;
3570
3571         if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through) {
3572                 domain_remove_one_dev_info(domain, pdev);
3573
3574                 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3575                     !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
3576                     list_empty(&domain->devices))
3577                         domain_exit(domain);
3578         }
3579
3580         return 0;
3581 }
3582
3583 static struct notifier_block device_nb = {
3584         .notifier_call = device_notifier,
3585 };
3586
3587 int __init intel_iommu_init(void)
3588 {
3589         int ret = 0;
3590
3591         /* VT-d is required for a TXT/tboot launch, so enforce that */
3592         force_on = tboot_force_iommu();
3593
3594         if (dmar_table_init()) {
3595                 if (force_on)
3596                         panic("tboot: Failed to initialize DMAR table\n");
3597                 return  -ENODEV;
3598         }
3599
3600         if (dmar_dev_scope_init() < 0) {
3601                 if (force_on)
3602                         panic("tboot: Failed to initialize DMAR device scope\n");
3603                 return  -ENODEV;
3604         }
3605
3606         if (no_iommu || dmar_disabled)
3607                 return -ENODEV;
3608
3609         if (iommu_init_mempool()) {
3610                 if (force_on)
3611                         panic("tboot: Failed to initialize iommu memory\n");
3612                 return  -ENODEV;
3613         }
3614
3615         if (list_empty(&dmar_rmrr_units))
3616                 printk(KERN_INFO "DMAR: No RMRR found\n");
3617
3618         if (list_empty(&dmar_atsr_units))
3619                 printk(KERN_INFO "DMAR: No ATSR found\n");
3620
3621         if (dmar_init_reserved_ranges()) {
3622                 if (force_on)
3623                         panic("tboot: Failed to reserve iommu ranges\n");
3624                 return  -ENODEV;
3625         }
3626
3627         init_no_remapping_devices();
3628
3629         ret = init_dmars();
3630         if (ret) {
3631                 if (force_on)
3632                         panic("tboot: Failed to initialize DMARs\n");
3633                 printk(KERN_ERR "IOMMU: dmar init failed\n");
3634                 put_iova_domain(&reserved_iova_list);
3635                 iommu_exit_mempool();
3636                 return ret;
3637         }
3638         printk(KERN_INFO
3639         "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3640
3641         init_timer(&unmap_timer);
3642 #ifdef CONFIG_SWIOTLB
3643         swiotlb = 0;
3644 #endif
3645         dma_ops = &intel_dma_ops;
3646
3647         init_iommu_pm_ops();
3648
3649         bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
3650
3651         bus_register_notifier(&pci_bus_type, &device_nb);
3652
3653         intel_iommu_enabled = 1;
3654
3655         return 0;
3656 }
3657
3658 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3659                                            struct pci_dev *pdev)
3660 {
3661         struct pci_dev *tmp, *parent;
3662
3663         if (!iommu || !pdev)
3664                 return;
3665
3666         /* dependent device detach */
3667         tmp = pci_find_upstream_pcie_bridge(pdev);
3668         /* Secondary interface's bus number and devfn 0 */
3669         if (tmp) {
3670                 parent = pdev->bus->self;
3671                 while (parent != tmp) {
3672                         iommu_detach_dev(iommu, parent->bus->number,
3673                                          parent->devfn);
3674                         parent = parent->bus->self;
3675                 }
3676                 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
3677                         iommu_detach_dev(iommu,
3678                                 tmp->subordinate->number, 0);
3679                 else /* this is a legacy PCI bridge */
3680                         iommu_detach_dev(iommu, tmp->bus->number,
3681                                          tmp->devfn);
3682         }
3683 }
3684
3685 static void domain_remove_one_dev_info(struct dmar_domain *domain,
3686                                           struct pci_dev *pdev)
3687 {
3688         struct device_domain_info *info;
3689         struct intel_iommu *iommu;
3690         unsigned long flags;
3691         int found = 0;
3692         struct list_head *entry, *tmp;
3693
3694         iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3695                                 pdev->devfn);
3696         if (!iommu)
3697                 return;
3698
3699         spin_lock_irqsave(&device_domain_lock, flags);
3700         list_for_each_safe(entry, tmp, &domain->devices) {
3701                 info = list_entry(entry, struct device_domain_info, link);
3702                 if (info->segment == pci_domain_nr(pdev->bus) &&
3703                     info->bus == pdev->bus->number &&
3704                     info->devfn == pdev->devfn) {
3705                         list_del(&info->link);
3706                         list_del(&info->global);
3707                         if (info->dev)
3708                                 info->dev->dev.archdata.iommu = NULL;
3709                         spin_unlock_irqrestore(&device_domain_lock, flags);
3710
3711                         iommu_disable_dev_iotlb(info);
3712                         iommu_detach_dev(iommu, info->bus, info->devfn);
3713                         iommu_detach_dependent_devices(iommu, pdev);
3714                         free_devinfo_mem(info);
3715
3716                         spin_lock_irqsave(&device_domain_lock, flags);
3717
3718                         if (found)
3719                                 break;
3720                         else
3721                                 continue;
3722                 }
3723
3724                 /* if there is no other devices under the same iommu
3725                  * owned by this domain, clear this iommu in iommu_bmp
3726                  * update iommu count and coherency
3727                  */
3728                 if (iommu == device_to_iommu(info->segment, info->bus,
3729                                             info->devfn))
3730                         found = 1;
3731         }
3732
3733         spin_unlock_irqrestore(&device_domain_lock, flags);
3734
3735         if (found == 0) {
3736                 unsigned long tmp_flags;
3737                 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
3738                 clear_bit(iommu->seq_id, &domain->iommu_bmp);
3739                 domain->iommu_count--;
3740                 domain_update_iommu_cap(domain);
3741                 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
3742
3743                 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3744                     !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)) {
3745                         spin_lock_irqsave(&iommu->lock, tmp_flags);
3746                         clear_bit(domain->id, iommu->domain_ids);
3747                         iommu->domains[domain->id] = NULL;
3748                         spin_unlock_irqrestore(&iommu->lock, tmp_flags);
3749                 }
3750         }
3751 }
3752
3753 static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3754 {
3755         struct device_domain_info *info;
3756         struct intel_iommu *iommu;
3757         unsigned long flags1, flags2;
3758
3759         spin_lock_irqsave(&device_domain_lock, flags1);
3760         while (!list_empty(&domain->devices)) {
3761                 info = list_entry(domain->devices.next,
3762                         struct device_domain_info, link);
3763                 list_del(&info->link);
3764                 list_del(&info->global);
3765                 if (info->dev)
3766                         info->dev->dev.archdata.iommu = NULL;
3767
3768                 spin_unlock_irqrestore(&device_domain_lock, flags1);
3769
3770                 iommu_disable_dev_iotlb(info);
3771                 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
3772                 iommu_detach_dev(iommu, info->bus, info->devfn);
3773                 iommu_detach_dependent_devices(iommu, info->dev);
3774
3775                 /* clear this iommu in iommu_bmp, update iommu count
3776                  * and capabilities
3777                  */
3778                 spin_lock_irqsave(&domain->iommu_lock, flags2);
3779                 if (test_and_clear_bit(iommu->seq_id,
3780                                        &domain->iommu_bmp)) {
3781                         domain->iommu_count--;
3782                         domain_update_iommu_cap(domain);
3783                 }
3784                 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3785
3786                 free_devinfo_mem(info);
3787                 spin_lock_irqsave(&device_domain_lock, flags1);
3788         }
3789         spin_unlock_irqrestore(&device_domain_lock, flags1);
3790 }
3791
3792 /* domain id for virtual machine, it won't be set in context */
3793 static unsigned long vm_domid;
3794
3795 static struct dmar_domain *iommu_alloc_vm_domain(void)
3796 {
3797         struct dmar_domain *domain;
3798
3799         domain = alloc_domain_mem();
3800         if (!domain)
3801                 return NULL;
3802
3803         domain->id = vm_domid++;
3804         domain->nid = -1;
3805         memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
3806         domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3807
3808         return domain;
3809 }
3810
3811 static int md_domain_init(struct dmar_domain *domain, int guest_width)
3812 {
3813         int adjust_width;
3814
3815         init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
3816         spin_lock_init(&domain->iommu_lock);
3817
3818         domain_reserve_special_ranges(domain);
3819
3820         /* calculate AGAW */
3821         domain->gaw = guest_width;
3822         adjust_width = guestwidth_to_adjustwidth(guest_width);
3823         domain->agaw = width_to_agaw(adjust_width);
3824
3825         INIT_LIST_HEAD(&domain->devices);
3826
3827         domain->iommu_count = 0;
3828         domain->iommu_coherency = 0;
3829         domain->iommu_snooping = 0;
3830         domain->iommu_superpage = 0;
3831         domain->max_addr = 0;
3832         domain->nid = -1;
3833
3834         /* always allocate the top pgd */
3835         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
3836         if (!domain->pgd)
3837                 return -ENOMEM;
3838         domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3839         return 0;
3840 }
3841
3842 static void iommu_free_vm_domain(struct dmar_domain *domain)
3843 {
3844         unsigned long flags;
3845         struct dmar_drhd_unit *drhd;
3846         struct intel_iommu *iommu;
3847         unsigned long i;
3848         unsigned long ndomains;
3849
3850         for_each_drhd_unit(drhd) {
3851                 if (drhd->ignored)
3852                         continue;
3853                 iommu = drhd->iommu;
3854
3855                 ndomains = cap_ndoms(iommu->cap);
3856                 for_each_set_bit(i, iommu->domain_ids, ndomains) {
3857                         if (iommu->domains[i] == domain) {
3858                                 spin_lock_irqsave(&iommu->lock, flags);
3859                                 clear_bit(i, iommu->domain_ids);
3860                                 iommu->domains[i] = NULL;
3861                                 spin_unlock_irqrestore(&iommu->lock, flags);
3862                                 break;
3863                         }
3864                 }
3865         }
3866 }
3867
3868 static void vm_domain_exit(struct dmar_domain *domain)
3869 {
3870         /* Domain 0 is reserved, so dont process it */
3871         if (!domain)
3872                 return;
3873
3874         vm_domain_remove_all_dev_info(domain);
3875         /* destroy iovas */
3876         put_iova_domain(&domain->iovad);
3877
3878         /* clear ptes */
3879         dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3880
3881         /* free page tables */
3882         dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3883
3884         iommu_free_vm_domain(domain);
3885         free_domain_mem(domain);
3886 }
3887
3888 static int intel_iommu_domain_init(struct iommu_domain *domain)
3889 {
3890         struct dmar_domain *dmar_domain;
3891
3892         dmar_domain = iommu_alloc_vm_domain();
3893         if (!dmar_domain) {
3894                 printk(KERN_ERR
3895                         "intel_iommu_domain_init: dmar_domain == NULL\n");
3896                 return -ENOMEM;
3897         }
3898         if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
3899                 printk(KERN_ERR
3900                         "intel_iommu_domain_init() failed\n");
3901                 vm_domain_exit(dmar_domain);
3902                 return -ENOMEM;
3903         }
3904         domain_update_iommu_cap(dmar_domain);
3905         domain->priv = dmar_domain;
3906
3907         return 0;
3908 }
3909
3910 static void intel_iommu_domain_destroy(struct iommu_domain *domain)
3911 {
3912         struct dmar_domain *dmar_domain = domain->priv;
3913
3914         domain->priv = NULL;
3915         vm_domain_exit(dmar_domain);
3916 }
3917
3918 static int intel_iommu_attach_device(struct iommu_domain *domain,
3919                                      struct device *dev)
3920 {
3921         struct dmar_domain *dmar_domain = domain->priv;
3922         struct pci_dev *pdev = to_pci_dev(dev);
3923         struct intel_iommu *iommu;
3924         int addr_width;
3925
3926         /* normally pdev is not mapped */
3927         if (unlikely(domain_context_mapped(pdev))) {
3928                 struct dmar_domain *old_domain;
3929
3930                 old_domain = find_domain(pdev);
3931                 if (old_domain) {
3932                         if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
3933                             dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
3934                                 domain_remove_one_dev_info(old_domain, pdev);
3935                         else
3936                                 domain_remove_dev_info(old_domain);
3937                 }
3938         }
3939
3940         iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3941                                 pdev->devfn);
3942         if (!iommu)
3943                 return -ENODEV;
3944
3945         /* check if this iommu agaw is sufficient for max mapped address */
3946         addr_width = agaw_to_width(iommu->agaw);
3947         if (addr_width > cap_mgaw(iommu->cap))
3948                 addr_width = cap_mgaw(iommu->cap);
3949
3950         if (dmar_domain->max_addr > (1LL << addr_width)) {
3951                 printk(KERN_ERR "%s: iommu width (%d) is not "
3952                        "sufficient for the mapped address (%llx)\n",
3953                        __func__, addr_width, dmar_domain->max_addr);
3954                 return -EFAULT;
3955         }
3956         dmar_domain->gaw = addr_width;
3957
3958         /*
3959          * Knock out extra levels of page tables if necessary
3960          */
3961         while (iommu->agaw < dmar_domain->agaw) {
3962                 struct dma_pte *pte;
3963
3964                 pte = dmar_domain->pgd;
3965                 if (dma_pte_present(pte)) {
3966                         dmar_domain->pgd = (struct dma_pte *)
3967                                 phys_to_virt(dma_pte_addr(pte));
3968                         free_pgtable_page(pte);
3969                 }
3970                 dmar_domain->agaw--;
3971         }
3972
3973         return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
3974 }
3975
3976 static void intel_iommu_detach_device(struct iommu_domain *domain,
3977                                       struct device *dev)
3978 {
3979         struct dmar_domain *dmar_domain = domain->priv;
3980         struct pci_dev *pdev = to_pci_dev(dev);
3981
3982         domain_remove_one_dev_info(dmar_domain, pdev);
3983 }
3984
3985 static int intel_iommu_map(struct iommu_domain *domain,
3986                            unsigned long iova, phys_addr_t hpa,
3987                            int gfp_order, int iommu_prot)
3988 {
3989         struct dmar_domain *dmar_domain = domain->priv;
3990         u64 max_addr;
3991         int prot = 0;
3992         size_t size;
3993         int ret;
3994
3995         if (iommu_prot & IOMMU_READ)
3996                 prot |= DMA_PTE_READ;
3997         if (iommu_prot & IOMMU_WRITE)
3998                 prot |= DMA_PTE_WRITE;
3999         if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4000                 prot |= DMA_PTE_SNP;
4001
4002         size     = PAGE_SIZE << gfp_order;
4003         max_addr = iova + size;
4004         if (dmar_domain->max_addr < max_addr) {
4005                 u64 end;
4006
4007                 /* check if minimum agaw is sufficient for mapped address */
4008                 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
4009                 if (end < max_addr) {
4010                         printk(KERN_ERR "%s: iommu width (%d) is not "
4011                                "sufficient for the mapped address (%llx)\n",
4012                                __func__, dmar_domain->gaw, max_addr);
4013                         return -EFAULT;
4014                 }
4015                 dmar_domain->max_addr = max_addr;
4016         }
4017         /* Round up size to next multiple of PAGE_SIZE, if it and
4018            the low bits of hpa would take us onto the next page */
4019         size = aligned_nrpages(hpa, size);
4020         ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4021                                  hpa >> VTD_PAGE_SHIFT, size, prot);
4022         return ret;
4023 }
4024
4025 static int intel_iommu_unmap(struct iommu_domain *domain,
4026                              unsigned long iova, int gfp_order)
4027 {
4028         struct dmar_domain *dmar_domain = domain->priv;
4029         size_t size = PAGE_SIZE << gfp_order;
4030         int order;
4031
4032         order = dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
4033                             (iova + size - 1) >> VTD_PAGE_SHIFT);
4034
4035         if (dmar_domain->max_addr == iova + size)
4036                 dmar_domain->max_addr = iova;
4037
4038         return order;
4039 }
4040
4041 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
4042                                             unsigned long iova)
4043 {
4044         struct dmar_domain *dmar_domain = domain->priv;
4045         struct dma_pte *pte;
4046         u64 phys = 0;
4047
4048         pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, 0);
4049         if (pte)
4050                 phys = dma_pte_addr(pte);
4051
4052         return phys;
4053 }
4054
4055 static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
4056                                       unsigned long cap)
4057 {
4058         struct dmar_domain *dmar_domain = domain->priv;
4059
4060         if (cap == IOMMU_CAP_CACHE_COHERENCY)
4061                 return dmar_domain->iommu_snooping;
4062         if (cap == IOMMU_CAP_INTR_REMAP)
4063                 return intr_remapping_enabled;
4064
4065         return 0;
4066 }
4067
4068 static struct iommu_ops intel_iommu_ops = {
4069         .domain_init    = intel_iommu_domain_init,
4070         .domain_destroy = intel_iommu_domain_destroy,
4071         .attach_dev     = intel_iommu_attach_device,
4072         .detach_dev     = intel_iommu_detach_device,
4073         .map            = intel_iommu_map,
4074         .unmap          = intel_iommu_unmap,
4075         .iova_to_phys   = intel_iommu_iova_to_phys,
4076         .domain_has_cap = intel_iommu_domain_has_cap,
4077 };
4078
4079 static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
4080 {
4081         /*
4082          * Mobile 4 Series Chipset neglects to set RWBF capability,
4083          * but needs it:
4084          */
4085         printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
4086         rwbf_quirk = 1;
4087
4088         /* https://bugzilla.redhat.com/show_bug.cgi?id=538163 */
4089         if (dev->revision == 0x07) {
4090                 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
4091                 dmar_map_gfx = 0;
4092         }
4093 }
4094
4095 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
4096
4097 #define GGC 0x52
4098 #define GGC_MEMORY_SIZE_MASK    (0xf << 8)
4099 #define GGC_MEMORY_SIZE_NONE    (0x0 << 8)
4100 #define GGC_MEMORY_SIZE_1M      (0x1 << 8)
4101 #define GGC_MEMORY_SIZE_2M      (0x3 << 8)
4102 #define GGC_MEMORY_VT_ENABLED   (0x8 << 8)
4103 #define GGC_MEMORY_SIZE_2M_VT   (0x9 << 8)
4104 #define GGC_MEMORY_SIZE_3M_VT   (0xa << 8)
4105 #define GGC_MEMORY_SIZE_4M_VT   (0xb << 8)
4106
4107 static void __devinit quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
4108 {
4109         unsigned short ggc;
4110
4111         if (pci_read_config_word(dev, GGC, &ggc))
4112                 return;
4113
4114         if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
4115                 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4116                 dmar_map_gfx = 0;
4117         } else if (dmar_map_gfx) {
4118                 /* we have to ensure the gfx device is idle before we flush */
4119                 printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");
4120                 intel_iommu_strict = 1;
4121        }
4122 }
4123 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4124 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4125 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4126 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4127
4128 /* On Tylersburg chipsets, some BIOSes have been known to enable the
4129    ISOCH DMAR unit for the Azalia sound device, but not give it any
4130    TLB entries, which causes it to deadlock. Check for that.  We do
4131    this in a function called from init_dmars(), instead of in a PCI
4132    quirk, because we don't want to print the obnoxious "BIOS broken"
4133    message if VT-d is actually disabled.
4134 */
4135 static void __init check_tylersburg_isoch(void)
4136 {
4137         struct pci_dev *pdev;
4138         uint32_t vtisochctrl;
4139
4140         /* If there's no Azalia in the system anyway, forget it. */
4141         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4142         if (!pdev)
4143                 return;
4144         pci_dev_put(pdev);
4145
4146         /* System Management Registers. Might be hidden, in which case
4147            we can't do the sanity check. But that's OK, because the
4148            known-broken BIOSes _don't_ actually hide it, so far. */
4149         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4150         if (!pdev)
4151                 return;
4152
4153         if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4154                 pci_dev_put(pdev);
4155                 return;
4156         }
4157
4158         pci_dev_put(pdev);
4159
4160         /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4161         if (vtisochctrl & 1)
4162                 return;
4163
4164         /* Drop all bits other than the number of TLB entries */
4165         vtisochctrl &= 0x1c;
4166
4167         /* If we have the recommended number of TLB entries (16), fine. */
4168         if (vtisochctrl == 0x10)
4169                 return;
4170
4171         /* Zero TLB entries? You get to ride the short bus to school. */
4172         if (!vtisochctrl) {
4173                 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4174                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4175                      dmi_get_system_info(DMI_BIOS_VENDOR),
4176                      dmi_get_system_info(DMI_BIOS_VERSION),
4177                      dmi_get_system_info(DMI_PRODUCT_VERSION));
4178                 iommu_identity_mapping |= IDENTMAP_AZALIA;
4179                 return;
4180         }
4181
4182         printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4183                vtisochctrl);
4184 }