drivers/iommu/intel-iommu.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  * Author: Fenghua Yu <fenghua.yu@intel.com>
  22  */
  23
  24 #include <linux/init.h>
  25 #include <linux/bitmap.h>
  26 #include <linux/debugfs.h>
  27 #include <linux/export.h>
  28 #include <linux/slab.h>
  29 #include <linux/irq.h>
  30 #include <linux/interrupt.h>
  31 #include <linux/spinlock.h>
  32 #include <linux/pci.h>
  33 #include <linux/dmar.h>
  34 #include <linux/dma-mapping.h>
  35 #include <linux/mempool.h>
  36 #include <linux/timer.h>
  37 #include <linux/iova.h>
  38 #include <linux/iommu.h>
  39 #include <linux/intel-iommu.h>
  40 #include <linux/syscore_ops.h>
  41 #include <linux/tboot.h>
  42 #include <linux/dmi.h>
  43 #include <linux/pci-ats.h>
  44 #include <asm/cacheflush.h>
  45 #include <asm/iommu.h>
  46
  47 #define ROOT_SIZE               VTD_PAGE_SIZE
  48 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  49
  50 #define IS_BRIDGE_HOST_DEVICE(pdev) \
  51                             ((pdev->class >> 8) == PCI_CLASS_BRIDGE_HOST)
  52 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  53 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  54 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
  55
  56 #define IOAPIC_RANGE_START      (0xfee00000)
  57 #define IOAPIC_RANGE_END        (0xfeefffff)
  58 #define IOVA_START_ADDR         (0x1000)
  59
  60 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
  61
  62 #define MAX_AGAW_WIDTH 64
  63
  64 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
  65 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
  66
  67 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
  68    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
  69 #define DOMAIN_MAX_PFN(gaw)     ((unsigned long) min_t(uint64_t, \
  70                                 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
  71 #define DOMAIN_MAX_ADDR(gaw)    (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
  72
  73 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  74 #define DMA_32BIT_PFN           IOVA_PFN(DMA_BIT_MASK(32))
  75 #define DMA_64BIT_PFN           IOVA_PFN(DMA_BIT_MASK(64))
  76
  77 /* page table handling */
  78 #define LEVEL_STRIDE            (9)
  79 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
  80
  81 static inline int agaw_to_level(int agaw)
  82 {
  83         return agaw + 2;
  84 }
  85
  86 static inline int agaw_to_width(int agaw)
  87 {
  88         return 30 + agaw * LEVEL_STRIDE;
  89 }
  90
  91 static inline int width_to_agaw(int width)
  92 {
  93         return (width - 30) / LEVEL_STRIDE;
  94 }
  95
  96 static inline unsigned int level_to_offset_bits(int level)
  97 {
  98         return (level - 1) * LEVEL_STRIDE;
  99 }
 100
 101 static inline int pfn_level_offset(unsigned long pfn, int level)
 102 {
 103         return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
 104 }
 105
 106 static inline unsigned long level_mask(int level)
 107 {
 108         return -1UL << level_to_offset_bits(level);
 109 }
 110
 111 static inline unsigned long level_size(int level)
 112 {
 113         return 1UL << level_to_offset_bits(level);
 114 }
 115
 116 static inline unsigned long align_to_level(unsigned long pfn, int level)
 117 {
 118         return (pfn + level_size(level) - 1) & level_mask(level);
 119 }
 120
 121 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
 122 {
 123         return  1 << ((lvl - 1) * LEVEL_STRIDE);
 124 }
 125
 126 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
 127    are never going to work. */
 128 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
 129 {
 130         return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
 131 }
 132
 133 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
 134 {
 135         return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
 136 }
 137 static inline unsigned long page_to_dma_pfn(struct page *pg)
 138 {
 139         return mm_to_dma_pfn(page_to_pfn(pg));
 140 }
 141 static inline unsigned long virt_to_dma_pfn(void *p)
 142 {
 143         return page_to_dma_pfn(virt_to_page(p));
 144 }
 145
 146 /* global iommu list, set NULL for ignored DMAR units */
 147 static struct intel_iommu **g_iommus;
 148
 149 static void __init check_tylersburg_isoch(void);
 150 static int rwbf_quirk;
 151
 152 /*
 153  * set to 1 to panic kernel if can't successfully enable VT-d
 154  * (used when kernel is launched w/ TXT)
 155  */
 156 static int force_on = 0;
 157
 158 /*
 159  * 0: Present
 160  * 1-11: Reserved
 161  * 12-63: Context Ptr (12 - (haw-1))
 162  * 64-127: Reserved
 163  */
 164 struct root_entry {
 165         u64     val;
 166         u64     rsvd1;
 167 };
 168 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
 169 static inline bool root_present(struct root_entry *root)
 170 {
 171         return (root->val & 1);
 172 }
 173 static inline void set_root_present(struct root_entry *root)
 174 {
 175         root->val |= 1;
 176 }
 177 static inline void set_root_value(struct root_entry *root, unsigned long value)
 178 {
 179         root->val |= value & VTD_PAGE_MASK;
 180 }
 181
 182 static inline struct context_entry *
 183 get_context_addr_from_root(struct root_entry *root)
 184 {
 185         return (struct context_entry *)
 186                 (root_present(root)?phys_to_virt(
 187                 root->val & VTD_PAGE_MASK) :
 188                 NULL);
 189 }
 190
 191 /*
 192  * low 64 bits:
 193  * 0: present
 194  * 1: fault processing disable
 195  * 2-3: translation type
 196  * 12-63: address space root
 197  * high 64 bits:
 198  * 0-2: address width
 199  * 3-6: aval
 200  * 8-23: domain id
 201  */
 202 struct context_entry {
 203         u64 lo;
 204         u64 hi;
 205 };
 206
 207 static inline bool context_present(struct context_entry *context)
 208 {
 209         return (context->lo & 1);
 210 }
 211 static inline void context_set_present(struct context_entry *context)
 212 {
 213         context->lo |= 1;
 214 }
 215
 216 static inline void context_set_fault_enable(struct context_entry *context)
 217 {
 218         context->lo &= (((u64)-1) << 2) | 1;
 219 }
 220
 221 static inline void context_set_translation_type(struct context_entry *context,
 222                                                 unsigned long value)
 223 {
 224         context->lo &= (((u64)-1) << 4) | 3;
 225         context->lo |= (value & 3) << 2;
 226 }
 227
 228 static inline void context_set_address_root(struct context_entry *context,
 229                                             unsigned long value)
 230 {
 231         context->lo |= value & VTD_PAGE_MASK;
 232 }
 233
 234 static inline void context_set_address_width(struct context_entry *context,
 235                                              unsigned long value)
 236 {
 237         context->hi |= value & 7;
 238 }
 239
 240 static inline void context_set_domain_id(struct context_entry *context,
 241                                          unsigned long value)
 242 {
 243         context->hi |= (value & ((1 << 16) - 1)) << 8;
 244 }
 245
 246 static inline void context_clear_entry(struct context_entry *context)
 247 {
 248         context->lo = 0;
 249         context->hi = 0;
 250 }
 251
 252 /*
 253  * 0: readable
 254  * 1: writable
 255  * 2-6: reserved
 256  * 7: super page
 257  * 8-10: available
 258  * 11: snoop behavior
 259  * 12-63: Host physcial address
 260  */
 261 struct dma_pte {
 262         u64 val;
 263 };
 264
 265 static inline void dma_clear_pte(struct dma_pte *pte)
 266 {
 267         pte->val = 0;
 268 }
 269
 270 static inline void dma_set_pte_readable(struct dma_pte *pte)
 271 {
 272         pte->val |= DMA_PTE_READ;
 273 }
 274
 275 static inline void dma_set_pte_writable(struct dma_pte *pte)
 276 {
 277         pte->val |= DMA_PTE_WRITE;
 278 }
 279
 280 static inline void dma_set_pte_snp(struct dma_pte *pte)
 281 {
 282         pte->val |= DMA_PTE_SNP;
 283 }
 284
 285 static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
 286 {
 287         pte->val = (pte->val & ~3) | (prot & 3);
 288 }
 289
 290 static inline u64 dma_pte_addr(struct dma_pte *pte)
 291 {
 292 #ifdef CONFIG_64BIT
 293         return pte->val & VTD_PAGE_MASK;
 294 #else
 295         /* Must have a full atomic 64-bit read */
 296         return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
 297 #endif
 298 }
 299
 300 static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
 301 {
 302         pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
 303 }
 304
 305 static inline bool dma_pte_present(struct dma_pte *pte)
 306 {
 307         return (pte->val & 3) != 0;
 308 }
 309
 310 static inline bool dma_pte_superpage(struct dma_pte *pte)
 311 {
 312         return (pte->val & (1 << 7));
 313 }
 314
 315 static inline int first_pte_in_page(struct dma_pte *pte)
 316 {
 317         return !((unsigned long)pte & ~VTD_PAGE_MASK);
 318 }
 319
 320 /*
 321  * This domain is a statically identity mapping domain.
 322  *      1. This domain creats a static 1:1 mapping to all usable memory.
 323  *      2. It maps to each iommu if successful.
 324  *      3. Each iommu mapps to this domain if successful.
 325  */
 326 static struct dmar_domain *si_domain;
 327 static int hw_pass_through = 1;
 328
 329 /* devices under the same p2p bridge are owned in one domain */
 330 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
 331
 332 /* domain represents a virtual machine, more than one devices
 333  * across iommus may be owned in one domain, e.g. kvm guest.
 334  */
 335 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 1)
 336
 337 /* si_domain contains mulitple devices */
 338 #define DOMAIN_FLAG_STATIC_IDENTITY     (1 << 2)
 339
 340 struct dmar_domain {
 341         int     id;                     /* domain id */
 342         int     nid;                    /* node id */
 343         unsigned long iommu_bmp;        /* bitmap of iommus this domain uses*/
 344
 345         struct list_head devices;       /* all devices' list */
 346         struct iova_domain iovad;       /* iova's that belong to this domain */
 347
 348         struct dma_pte  *pgd;           /* virtual address */
 349         int             gaw;            /* max guest address width */
 350
 351         /* adjusted guest address width, 0 is level 2 30-bit */
 352         int             agaw;
 353
 354         int             flags;          /* flags to find out type of domain */
 355
 356         int             iommu_coherency;/* indicate coherency of iommu access */
 357         int             iommu_snooping; /* indicate snooping control feature*/
 358         int             iommu_count;    /* reference count of iommu */
 359         int             iommu_superpage;/* Level of superpages supported:
 360                                            0 == 4KiB (no superpages), 1 == 2MiB,
 361                                            2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
 362         spinlock_t      iommu_lock;     /* protect iommu set in domain */
 363         u64             max_addr;       /* maximum mapped address */
 364 };
 365
 366 /* PCI domain-device relationship */
 367 struct device_domain_info {
 368         struct list_head link;  /* link to domain siblings */
 369         struct list_head global; /* link to global list */
 370         int segment;            /* PCI domain */
 371         u8 bus;                 /* PCI bus number */
 372         u8 devfn;               /* PCI devfn number */
 373         struct pci_dev *dev; /* it's NULL for PCIe-to-PCI bridge */
 374         struct intel_iommu *iommu; /* IOMMU used by this device */
 375         struct dmar_domain *domain; /* pointer to domain */
 376 };
 377
 378 static void flush_unmaps_timeout(unsigned long data);
 379
 380 DEFINE_TIMER(unmap_timer,  flush_unmaps_timeout, 0, 0);
 381
 382 #define HIGH_WATER_MARK 250
 383 struct deferred_flush_tables {
 384         int next;
 385         struct iova *iova[HIGH_WATER_MARK];
 386         struct dmar_domain *domain[HIGH_WATER_MARK];
 387 };
 388
 389 static struct deferred_flush_tables *deferred_flush;
 390
 391 /* bitmap for indexing intel_iommus */
 392 static int g_num_of_iommus;
 393
 394 static DEFINE_SPINLOCK(async_umap_flush_lock);
 395 static LIST_HEAD(unmaps_to_do);
 396
 397 static int timer_on;
 398 static long list_size;
 399
 400 static void domain_remove_dev_info(struct dmar_domain *domain);
 401
 402 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
 403 int dmar_disabled = 0;
 404 #else
 405 int dmar_disabled = 1;
 406 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
 407
 408 static int dmar_map_gfx = 1;
 409 static int dmar_forcedac;
 410 static int intel_iommu_strict;
 411 static int intel_iommu_superpage = 1;
 412
 413 int intel_iommu_gfx_mapped;
 414 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
 415
 416 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 417 static DEFINE_SPINLOCK(device_domain_lock);
 418 static LIST_HEAD(device_domain_list);
 419
 420 static struct iommu_ops intel_iommu_ops;
 421
 422 static int __init intel_iommu_setup(char *str)
 423 {
 424         if (!str)
 425                 return -EINVAL;
 426         while (*str) {
 427                 if (!strncmp(str, "on", 2)) {
 428                         dmar_disabled = 0;
 429                         printk(KERN_INFO "Intel-IOMMU: enabled\n");
 430                 } else if (!strncmp(str, "off", 3)) {
 431                         dmar_disabled = 1;
 432                         printk(KERN_INFO "Intel-IOMMU: disabled\n");
 433                 } else if (!strncmp(str, "igfx_off", 8)) {
 434                         dmar_map_gfx = 0;
 435                         printk(KERN_INFO
 436                                 "Intel-IOMMU: disable GFX device mapping\n");
 437                 } else if (!strncmp(str, "forcedac", 8)) {
 438                         printk(KERN_INFO
 439                                 "Intel-IOMMU: Forcing DAC for PCI devices\n");
 440                         dmar_forcedac = 1;
 441                 } else if (!strncmp(str, "strict", 6)) {
 442                         printk(KERN_INFO
 443                                 "Intel-IOMMU: disable batched IOTLB flush\n");
 444                         intel_iommu_strict = 1;
 445                 } else if (!strncmp(str, "sp_off", 6)) {
 446                         printk(KERN_INFO
 447                                 "Intel-IOMMU: disable supported super page\n");
 448                         intel_iommu_superpage = 0;
 449                 }
 450
 451                 str += strcspn(str, ",");
 452                 while (*str == ',')
 453                         str++;
 454         }
 455         return 0;
 456 }
 457 __setup("intel_iommu=", intel_iommu_setup);
 458
 459 static struct kmem_cache *iommu_domain_cache;
 460 static struct kmem_cache *iommu_devinfo_cache;
 461 static struct kmem_cache *iommu_iova_cache;
 462
 463 static inline void *alloc_pgtable_page(int node)
 464 {
 465         struct page *page;
 466         void *vaddr = NULL;
 467
 468         page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
 469         if (page)
 470                 vaddr = page_address(page);
 471         return vaddr;
 472 }
 473
 474 static inline void free_pgtable_page(void *vaddr)
 475 {
 476         free_page((unsigned long)vaddr);
 477 }
 478
 479 static inline void *alloc_domain_mem(void)
 480 {
 481         return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
 482 }
 483
 484 static void free_domain_mem(void *vaddr)
 485 {
 486         kmem_cache_free(iommu_domain_cache, vaddr);
 487 }
 488
 489 static inline void * alloc_devinfo_mem(void)
 490 {
 491         return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
 492 }
 493
 494 static inline void free_devinfo_mem(void *vaddr)
 495 {
 496         kmem_cache_free(iommu_devinfo_cache, vaddr);
 497 }
 498
 499 struct iova *alloc_iova_mem(void)
 500 {
 501         return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
 502 }
 503
 504 void free_iova_mem(struct iova *iova)
 505 {
 506         kmem_cache_free(iommu_iova_cache, iova);
 507 }
 508
 509
 510 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
 511 {
 512         unsigned long sagaw;
 513         int agaw = -1;
 514
 515         sagaw = cap_sagaw(iommu->cap);
 516         for (agaw = width_to_agaw(max_gaw);
 517              agaw >= 0; agaw--) {
 518                 if (test_bit(agaw, &sagaw))
 519                         break;
 520         }
 521
 522         return agaw;
 523 }
 524
 525 /*
 526  * Calculate max SAGAW for each iommu.
 527  */
 528 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
 529 {
 530         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
 531 }
 532
 533 /*
 534  * calculate agaw for each iommu.
 535  * "SAGAW" may be different across iommus, use a default agaw, and
 536  * get a supported less agaw for iommus that don't support the default agaw.
 537  */
 538 int iommu_calculate_agaw(struct intel_iommu *iommu)
 539 {
 540         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 541 }
 542
 543 /* This functionin only returns single iommu in a domain */
 544 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 545 {
 546         int iommu_id;
 547
 548         /* si_domain and vm domain should not get here. */
 549         BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
 550         BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
 551
 552         iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
 553         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 554                 return NULL;
 555
 556         return g_iommus[iommu_id];
 557 }
 558
 559 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 560 {
 561         int i;
 562
 563         domain->iommu_coherency = 1;
 564
 565         for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
 566                 if (!ecap_coherent(g_iommus[i]->ecap)) {
 567                         domain->iommu_coherency = 0;
 568                         break;
 569                 }
 570         }
 571 }
 572
 573 static void domain_update_iommu_snooping(struct dmar_domain *domain)
 574 {
 575         int i;
 576
 577         domain->iommu_snooping = 1;
 578
 579         for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
 580                 if (!ecap_sc_support(g_iommus[i]->ecap)) {
 581                         domain->iommu_snooping = 0;
 582                         break;
 583                 }
 584         }
 585 }
 586
 587 static void domain_update_iommu_superpage(struct dmar_domain *domain)
 588 {
 589         struct dmar_drhd_unit *drhd;
 590         struct intel_iommu *iommu = NULL;
 591         int mask = 0xf;
 592
 593         if (!intel_iommu_superpage) {
 594                 domain->iommu_superpage = 0;
 595                 return;
 596         }
 597
 598         /* set iommu_superpage to the smallest common denominator */
 599         for_each_active_iommu(iommu, drhd) {
 600                 mask &= cap_super_page_val(iommu->cap);
 601                 if (!mask) {
 602                         break;
 603                 }
 604         }
 605         domain->iommu_superpage = fls(mask);
 606 }
 607
 608 /* Some capabilities may be different across iommus */
 609 static void domain_update_iommu_cap(struct dmar_domain *domain)
 610 {
 611         domain_update_iommu_coherency(domain);
 612         domain_update_iommu_snooping(domain);
 613         domain_update_iommu_superpage(domain);
 614 }
 615
 616 static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
 617 {
 618         struct dmar_drhd_unit *drhd = NULL;
 619         int i;
 620
 621         for_each_drhd_unit(drhd) {
 622                 if (drhd->ignored)
 623                         continue;
 624                 if (segment != drhd->segment)
 625                         continue;
 626
 627                 for (i = 0; i < drhd->devices_cnt; i++) {
 628                         if (drhd->devices[i] &&
 629                             drhd->devices[i]->bus->number == bus &&
 630                             drhd->devices[i]->devfn == devfn)
 631                                 return drhd->iommu;
 632                         if (drhd->devices[i] &&
 633                             drhd->devices[i]->subordinate &&
 634                             drhd->devices[i]->subordinate->number <= bus &&
 635                             drhd->devices[i]->subordinate->subordinate >= bus)
 636                                 return drhd->iommu;
 637                 }
 638
 639                 if (drhd->include_all)
 640                         return drhd->iommu;
 641         }
 642
 643         return NULL;
 644 }
 645
 646 static void domain_flush_cache(struct dmar_domain *domain,
 647                                void *addr, int size)
 648 {
 649         if (!domain->iommu_coherency)
 650                 clflush_cache_range(addr, size);
 651 }
 652
 653 /* Gets context entry for a given bus and devfn */
 654 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
 655                 u8 bus, u8 devfn)
 656 {
 657         struct root_entry *root;
 658         struct context_entry *context;
 659         unsigned long phy_addr;
 660         unsigned long flags;
 661
 662         spin_lock_irqsave(&iommu->lock, flags);
 663         root = &iommu->root_entry[bus];
 664         context = get_context_addr_from_root(root);
 665         if (!context) {
 666                 context = (struct context_entry *)
 667                                 alloc_pgtable_page(iommu->node);
 668                 if (!context) {
 669                         spin_unlock_irqrestore(&iommu->lock, flags);
 670                         return NULL;
 671                 }
 672                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 673                 phy_addr = virt_to_phys((void *)context);
 674                 set_root_value(root, phy_addr);
 675                 set_root_present(root);
 676                 __iommu_flush_cache(iommu, root, sizeof(*root));
 677         }
 678         spin_unlock_irqrestore(&iommu->lock, flags);
 679         return &context[devfn];
 680 }
 681
 682 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 683 {
 684         struct root_entry *root;
 685         struct context_entry *context;
 686         int ret;
 687         unsigned long flags;
 688
 689         spin_lock_irqsave(&iommu->lock, flags);
 690         root = &iommu->root_entry[bus];
 691         context = get_context_addr_from_root(root);
 692         if (!context) {
 693                 ret = 0;
 694                 goto out;
 695         }
 696         ret = context_present(&context[devfn]);
 697 out:
 698         spin_unlock_irqrestore(&iommu->lock, flags);
 699         return ret;
 700 }
 701
 702 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
 703 {
 704         struct root_entry *root;
 705         struct context_entry *context;
 706         unsigned long flags;
 707
 708         spin_lock_irqsave(&iommu->lock, flags);
 709         root = &iommu->root_entry[bus];
 710         context = get_context_addr_from_root(root);
 711         if (context) {
 712                 context_clear_entry(&context[devfn]);
 713                 __iommu_flush_cache(iommu, &context[devfn], \
 714                         sizeof(*context));
 715         }
 716         spin_unlock_irqrestore(&iommu->lock, flags);
 717 }
 718
 719 static void free_context_table(struct intel_iommu *iommu)
 720 {
 721         struct root_entry *root;
 722         int i;
 723         unsigned long flags;
 724         struct context_entry *context;
 725
 726         spin_lock_irqsave(&iommu->lock, flags);
 727         if (!iommu->root_entry) {
 728                 goto out;
 729         }
 730         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 731                 root = &iommu->root_entry[i];
 732                 context = get_context_addr_from_root(root);
 733                 if (context)
 734                         free_pgtable_page(context);
 735         }
 736         free_pgtable_page(iommu->root_entry);
 737         iommu->root_entry = NULL;
 738 out:
 739         spin_unlock_irqrestore(&iommu->lock, flags);
 740 }
 741
 742 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
 743                                       unsigned long pfn, int target_level)
 744 {
 745         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 746         struct dma_pte *parent, *pte = NULL;
 747         int level = agaw_to_level(domain->agaw);
 748         int offset;
 749
 750         BUG_ON(!domain->pgd);
 751         BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width);
 752         parent = domain->pgd;
 753
 754         while (level > 0) {
 755                 void *tmp_page;
 756
 757                 offset = pfn_level_offset(pfn, level);
 758                 pte = &parent[offset];
 759                 if (!target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
 760                         break;
 761                 if (level == target_level)
 762                         break;
 763
 764                 if (!dma_pte_present(pte)) {
 765                         uint64_t pteval;
 766
 767                         tmp_page = alloc_pgtable_page(domain->nid);
 768
 769                         if (!tmp_page)
 770                                 return NULL;
 771
 772                         domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
 773                         pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
 774                         if (cmpxchg64(&pte->val, 0ULL, pteval)) {
 775                                 /* Someone else set it while we were thinking; use theirs. */
 776                                 free_pgtable_page(tmp_page);
 777                         } else {
 778                                 dma_pte_addr(pte);
 779                                 domain_flush_cache(domain, pte, sizeof(*pte));
 780                         }
 781                 }
 782                 parent = phys_to_virt(dma_pte_addr(pte));
 783                 level--;
 784         }
 785
 786         return pte;
 787 }
 788
 789
 790 /* return address's pte at specific level */
 791 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
 792                                          unsigned long pfn,
 793                                          int level, int *large_page)
 794 {
 795         struct dma_pte *parent, *pte = NULL;
 796         int total = agaw_to_level(domain->agaw);
 797         int offset;
 798
 799         parent = domain->pgd;
 800         while (level <= total) {
 801                 offset = pfn_level_offset(pfn, total);
 802                 pte = &parent[offset];
 803                 if (level == total)
 804                         return pte;
 805
 806                 if (!dma_pte_present(pte)) {
 807                         *large_page = total;
 808                         break;
 809                 }
 810
 811                 if (pte->val & DMA_PTE_LARGE_PAGE) {
 812                         *large_page = total;
 813                         return pte;
 814                 }
 815
 816                 parent = phys_to_virt(dma_pte_addr(pte));
 817                 total--;
 818         }
 819         return NULL;
 820 }
 821
 822 /* clear last level pte, a tlb flush should be followed */
 823 static int dma_pte_clear_range(struct dmar_domain *domain,
 824                                 unsigned long start_pfn,
 825                                 unsigned long last_pfn)
 826 {
 827         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 828         unsigned int large_page = 1;
 829         struct dma_pte *first_pte, *pte;
 830         int order;
 831
 832         BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
 833         BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
 834         BUG_ON(start_pfn > last_pfn);
 835
 836         /* we don't need lock here; nobody else touches the iova range */
 837         do {
 838                 large_page = 1;
 839                 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
 840                 if (!pte) {
 841                         start_pfn = align_to_level(start_pfn + 1, large_page + 1);
 842                         continue;
 843                 }
 844                 do {
 845                         dma_clear_pte(pte);
 846                         start_pfn += lvl_to_nr_pages(large_page);
 847                         pte++;
 848                 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
 849
 850                 domain_flush_cache(domain, first_pte,
 851                                    (void *)pte - (void *)first_pte);
 852
 853         } while (start_pfn && start_pfn <= last_pfn);
 854
 855         order = (large_page - 1) * 9;
 856         return order;
 857 }
 858
 859 /* free page table pages. last level pte should already be cleared */
 860 static void dma_pte_free_pagetable(struct dmar_domain *domain,
 861                                    unsigned long start_pfn,
 862                                    unsigned long last_pfn)
 863 {
 864         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 865         struct dma_pte *first_pte, *pte;
 866         int total = agaw_to_level(domain->agaw);
 867         int level;
 868         unsigned long tmp;
 869         int large_page = 2;
 870
 871         BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
 872         BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
 873         BUG_ON(start_pfn > last_pfn);
 874
 875         /* We don't need lock here; nobody else touches the iova range */
 876         level = 2;
 877         while (level <= total) {
 878                 tmp = align_to_level(start_pfn, level);
 879
 880                 /* If we can't even clear one PTE at this level, we're done */
 881                 if (tmp + level_size(level) - 1 > last_pfn)
 882                         return;
 883
 884                 do {
 885                         large_page = level;
 886                         first_pte = pte = dma_pfn_level_pte(domain, tmp, level, &large_page);
 887                         if (large_page > level)
 888                                 level = large_page + 1;
 889                         if (!pte) {
 890                                 tmp = align_to_level(tmp + 1, level + 1);
 891                                 continue;
 892                         }
 893                         do {
 894                                 if (dma_pte_present(pte)) {
 895                                         free_pgtable_page(phys_to_virt(dma_pte_addr(pte)));
 896                                         dma_clear_pte(pte);
 897                                 }
 898                                 pte++;
 899                                 tmp += level_size(level);
 900                         } while (!first_pte_in_page(pte) &&
 901                                  tmp + level_size(level) - 1 <= last_pfn);
 902
 903                         domain_flush_cache(domain, first_pte,
 904                                            (void *)pte - (void *)first_pte);
 905
 906                 } while (tmp && tmp + level_size(level) - 1 <= last_pfn);
 907                 level++;
 908         }
 909         /* free pgd */
 910         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
 911                 free_pgtable_page(domain->pgd);
 912                 domain->pgd = NULL;
 913         }
 914 }
 915
 916 /* iommu handling */
 917 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
 918 {
 919         struct root_entry *root;
 920         unsigned long flags;
 921
 922         root = (struct root_entry *)alloc_pgtable_page(iommu->node);
 923         if (!root)
 924                 return -ENOMEM;
 925
 926         __iommu_flush_cache(iommu, root, ROOT_SIZE);
 927
 928         spin_lock_irqsave(&iommu->lock, flags);
 929         iommu->root_entry = root;
 930         spin_unlock_irqrestore(&iommu->lock, flags);
 931
 932         return 0;
 933 }
 934
 935 static void iommu_set_root_entry(struct intel_iommu *iommu)
 936 {
 937         void *addr;
 938         u32 sts;
 939         unsigned long flag;
 940
 941         addr = iommu->root_entry;
 942
 943         raw_spin_lock_irqsave(&iommu->register_lock, flag);
 944         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
 945
 946         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
 947
 948         /* Make sure hardware complete it */
 949         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 950                       readl, (sts & DMA_GSTS_RTPS), sts);
 951
 952         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
 953 }
 954
 955 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
 956 {
 957         u32 val;
 958         unsigned long flag;
 959
 960         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
 961                 return;
 962
 963         raw_spin_lock_irqsave(&iommu->register_lock, flag);
 964         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
 965
 966         /* Make sure hardware complete it */
 967         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 968                       readl, (!(val & DMA_GSTS_WBFS)), val);
 969
 970         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
 971 }
 972
 973 /* return value determine if we need a write buffer flush */
 974 static void __iommu_flush_context(struct intel_iommu *iommu,
 975                                   u16 did, u16 source_id, u8 function_mask,
 976                                   u64 type)
 977 {
 978         u64 val = 0;
 979         unsigned long flag;
 980
 981         switch (type) {
 982         case DMA_CCMD_GLOBAL_INVL:
 983                 val = DMA_CCMD_GLOBAL_INVL;
 984                 break;
 985         case DMA_CCMD_DOMAIN_INVL:
 986                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
 987                 break;
 988         case DMA_CCMD_DEVICE_INVL:
 989                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
 990                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
 991                 break;
 992         default:
 993                 BUG();
 994         }
 995         val |= DMA_CCMD_ICC;
 996
 997         raw_spin_lock_irqsave(&iommu->register_lock, flag);
 998         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
 999
1000         /* Make sure hardware complete it */
1001         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1002                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1003
1004         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1005 }
1006
1007 /* return value determine if we need a write buffer flush */
1008 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1009                                 u64 addr, unsigned int size_order, u64 type)
1010 {
1011         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1012         u64 val = 0, val_iva = 0;
1013         unsigned long flag;
1014
1015         switch (type) {
1016         case DMA_TLB_GLOBAL_FLUSH:
1017                 /* global flush doesn't need set IVA_REG */
1018                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1019                 break;
1020         case DMA_TLB_DSI_FLUSH:
1021                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1022                 break;
1023         case DMA_TLB_PSI_FLUSH:
1024                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1025                 /* Note: always flush non-leaf currently */
1026                 val_iva = size_order | addr;
1027                 break;
1028         default:
1029                 BUG();
1030         }
1031         /* Note: set drain read/write */
1032 #if 0
1033         /*
1034          * This is probably to be super secure.. Looks like we can
1035          * ignore it without any impact.
1036          */
1037         if (cap_read_drain(iommu->cap))
1038                 val |= DMA_TLB_READ_DRAIN;
1039 #endif
1040         if (cap_write_drain(iommu->cap))
1041                 val |= DMA_TLB_WRITE_DRAIN;
1042
1043         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1044         /* Note: Only uses first TLB reg currently */
1045         if (val_iva)
1046                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1047         dmar_writeq(iommu->reg + tlb_offset + 8, val);
1048
1049         /* Make sure hardware complete it */
1050         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1051                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1052
1053         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1054
1055         /* check IOTLB invalidation granularity */
1056         if (DMA_TLB_IAIG(val) == 0)
1057                 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1058         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1059                 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
1060                         (unsigned long long)DMA_TLB_IIRG(type),
1061                         (unsigned long long)DMA_TLB_IAIG(val));
1062 }
1063
1064 static struct device_domain_info *iommu_support_dev_iotlb(
1065         struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
1066 {
1067         int found = 0;
1068         unsigned long flags;
1069         struct device_domain_info *info;
1070         struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
1071
1072         if (!ecap_dev_iotlb_support(iommu->ecap))
1073                 return NULL;
1074
1075         if (!iommu->qi)
1076                 return NULL;
1077
1078         spin_lock_irqsave(&device_domain_lock, flags);
1079         list_for_each_entry(info, &domain->devices, link)
1080                 if (info->bus == bus && info->devfn == devfn) {
1081                         found = 1;
1082                         break;
1083                 }
1084         spin_unlock_irqrestore(&device_domain_lock, flags);
1085
1086         if (!found || !info->dev)
1087                 return NULL;
1088
1089         if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1090                 return NULL;
1091
1092         if (!dmar_find_matched_atsr_unit(info->dev))
1093                 return NULL;
1094
1095         info->iommu = iommu;
1096
1097         return info;
1098 }
1099
1100 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1101 {
1102         if (!info)
1103                 return;
1104
1105         pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1106 }
1107
1108 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1109 {
1110         if (!info->dev || !pci_ats_enabled(info->dev))
1111                 return;
1112
1113         pci_disable_ats(info->dev);
1114 }
1115
1116 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1117                                   u64 addr, unsigned mask)
1118 {
1119         u16 sid, qdep;
1120         unsigned long flags;
1121         struct device_domain_info *info;
1122
1123         spin_lock_irqsave(&device_domain_lock, flags);
1124         list_for_each_entry(info, &domain->devices, link) {
1125                 if (!info->dev || !pci_ats_enabled(info->dev))
1126                         continue;
1127
1128                 sid = info->bus << 8 | info->devfn;
1129                 qdep = pci_ats_queue_depth(info->dev);
1130                 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1131         }
1132         spin_unlock_irqrestore(&device_domain_lock, flags);
1133 }
1134
1135 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1136                                   unsigned long pfn, unsigned int pages, int map)
1137 {
1138         unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1139         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1140
1141         BUG_ON(pages == 0);
1142
1143         /*
1144          * Fallback to domain selective flush if no PSI support or the size is
1145          * too big.
1146          * PSI requires page size to be 2 ^ x, and the base address is naturally
1147          * aligned to the size
1148          */
1149         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1150                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1151                                                 DMA_TLB_DSI_FLUSH);
1152         else
1153                 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1154                                                 DMA_TLB_PSI_FLUSH);
1155
1156         /*
1157          * In caching mode, changes of pages from non-present to present require
1158          * flush. However, device IOTLB doesn't need to be flushed in this case.
1159          */
1160         if (!cap_caching_mode(iommu->cap) || !map)
1161                 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
1162 }
1163
1164 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1165 {
1166         u32 pmen;
1167         unsigned long flags;
1168
1169         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1170         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1171         pmen &= ~DMA_PMEN_EPM;
1172         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1173
1174         /* wait for the protected region status bit to clear */
1175         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1176                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1177
1178         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1179 }
1180
1181 static int iommu_enable_translation(struct intel_iommu *iommu)
1182 {
1183         u32 sts;
1184         unsigned long flags;
1185
1186         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1187         iommu->gcmd |= DMA_GCMD_TE;
1188         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1189
1190         /* Make sure hardware complete it */
1191         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1192                       readl, (sts & DMA_GSTS_TES), sts);
1193
1194         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1195         return 0;
1196 }
1197
1198 static int iommu_disable_translation(struct intel_iommu *iommu)
1199 {
1200         u32 sts;
1201         unsigned long flag;
1202
1203         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1204         iommu->gcmd &= ~DMA_GCMD_TE;
1205         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1206
1207         /* Make sure hardware complete it */
1208         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1209                       readl, (!(sts & DMA_GSTS_TES)), sts);
1210
1211         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1212         return 0;
1213 }
1214
1215
1216 static int iommu_init_domains(struct intel_iommu *iommu)
1217 {
1218         unsigned long ndomains;
1219         unsigned long nlongs;
1220
1221         ndomains = cap_ndoms(iommu->cap);
1222         pr_debug("IOMMU %d: Number of Domains supportd <%ld>\n", iommu->seq_id,
1223                         ndomains);
1224         nlongs = BITS_TO_LONGS(ndomains);
1225
1226         spin_lock_init(&iommu->lock);
1227
1228         /* TBD: there might be 64K domains,
1229          * consider other allocation for future chip
1230          */
1231         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1232         if (!iommu->domain_ids) {
1233                 printk(KERN_ERR "Allocating domain id array failed\n");
1234                 return -ENOMEM;
1235         }
1236         iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1237                         GFP_KERNEL);
1238         if (!iommu->domains) {
1239                 printk(KERN_ERR "Allocating domain array failed\n");
1240                 return -ENOMEM;
1241         }
1242
1243         /*
1244          * if Caching mode is set, then invalid translations are tagged
1245          * with domainid 0. Hence we need to pre-allocate it.
1246          */
1247         if (cap_caching_mode(iommu->cap))
1248                 set_bit(0, iommu->domain_ids);
1249         return 0;
1250 }
1251
1252
1253 static void domain_exit(struct dmar_domain *domain);
1254 static void vm_domain_exit(struct dmar_domain *domain);
1255
1256 void free_dmar_iommu(struct intel_iommu *iommu)
1257 {
1258         struct dmar_domain *domain;
1259         int i;
1260         unsigned long flags;
1261
1262         if ((iommu->domains) && (iommu->domain_ids)) {
1263                 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
1264                         domain = iommu->domains[i];
1265                         clear_bit(i, iommu->domain_ids);
1266
1267                         spin_lock_irqsave(&domain->iommu_lock, flags);
1268                         if (--domain->iommu_count == 0) {
1269                                 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1270                                         vm_domain_exit(domain);
1271                                 else
1272                                         domain_exit(domain);
1273                         }
1274                         spin_unlock_irqrestore(&domain->iommu_lock, flags);
1275                 }
1276         }
1277
1278         if (iommu->gcmd & DMA_GCMD_TE)
1279                 iommu_disable_translation(iommu);
1280
1281         if (iommu->irq) {
1282                 irq_set_handler_data(iommu->irq, NULL);
1283                 /* This will mask the irq */
1284                 free_irq(iommu->irq, iommu);
1285                 destroy_irq(iommu->irq);
1286         }
1287
1288         kfree(iommu->domains);
1289         kfree(iommu->domain_ids);
1290
1291         g_iommus[iommu->seq_id] = NULL;
1292
1293         /* if all iommus are freed, free g_iommus */
1294         for (i = 0; i < g_num_of_iommus; i++) {
1295                 if (g_iommus[i])
1296                         break;
1297         }
1298
1299         if (i == g_num_of_iommus)
1300                 kfree(g_iommus);
1301
1302         /* free context mapping */
1303         free_context_table(iommu);
1304 }
1305
1306 static struct dmar_domain *alloc_domain(void)
1307 {
1308         struct dmar_domain *domain;
1309
1310         domain = alloc_domain_mem();
1311         if (!domain)
1312                 return NULL;
1313
1314         domain->nid = -1;
1315         memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
1316         domain->flags = 0;
1317
1318         return domain;
1319 }
1320
1321 static int iommu_attach_domain(struct dmar_domain *domain,
1322                                struct intel_iommu *iommu)
1323 {
1324         int num;
1325         unsigned long ndomains;
1326         unsigned long flags;
1327
1328         ndomains = cap_ndoms(iommu->cap);
1329
1330         spin_lock_irqsave(&iommu->lock, flags);
1331
1332         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1333         if (num >= ndomains) {
1334                 spin_unlock_irqrestore(&iommu->lock, flags);
1335                 printk(KERN_ERR "IOMMU: no free domain ids\n");
1336                 return -ENOMEM;
1337         }
1338
1339         domain->id = num;
1340         set_bit(num, iommu->domain_ids);
1341         set_bit(iommu->seq_id, &domain->iommu_bmp);
1342         iommu->domains[num] = domain;
1343         spin_unlock_irqrestore(&iommu->lock, flags);
1344
1345         return 0;
1346 }
1347
1348 static void iommu_detach_domain(struct dmar_domain *domain,
1349                                 struct intel_iommu *iommu)
1350 {
1351         unsigned long flags;
1352         int num, ndomains;
1353         int found = 0;
1354
1355         spin_lock_irqsave(&iommu->lock, flags);
1356         ndomains = cap_ndoms(iommu->cap);
1357         for_each_set_bit(num, iommu->domain_ids, ndomains) {
1358                 if (iommu->domains[num] == domain) {
1359                         found = 1;
1360                         break;
1361                 }
1362         }
1363
1364         if (found) {
1365                 clear_bit(num, iommu->domain_ids);
1366                 clear_bit(iommu->seq_id, &domain->iommu_bmp);
1367                 iommu->domains[num] = NULL;
1368         }
1369         spin_unlock_irqrestore(&iommu->lock, flags);
1370 }
1371
1372 static struct iova_domain reserved_iova_list;
1373 static struct lock_class_key reserved_rbtree_key;
1374
1375 static int dmar_init_reserved_ranges(void)
1376 {
1377         struct pci_dev *pdev = NULL;
1378         struct iova *iova;
1379         int i;
1380
1381         init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1382
1383         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1384                 &reserved_rbtree_key);
1385
1386         /* IOAPIC ranges shouldn't be accessed by DMA */
1387         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1388                 IOVA_PFN(IOAPIC_RANGE_END));
1389         if (!iova) {
1390                 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1391                 return -ENODEV;
1392         }
1393
1394         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1395         for_each_pci_dev(pdev) {
1396                 struct resource *r;
1397
1398                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1399                         r = &pdev->resource[i];
1400                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1401                                 continue;
1402                         iova = reserve_iova(&reserved_iova_list,
1403                                             IOVA_PFN(r->start),
1404                                             IOVA_PFN(r->end));
1405                         if (!iova) {
1406                                 printk(KERN_ERR "Reserve iova failed\n");
1407                                 return -ENODEV;
1408                         }
1409                 }
1410         }
1411         return 0;
1412 }
1413
1414 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1415 {
1416         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1417 }
1418
1419 static inline int guestwidth_to_adjustwidth(int gaw)
1420 {
1421         int agaw;
1422         int r = (gaw - 12) % 9;
1423
1424         if (r == 0)
1425                 agaw = gaw;
1426         else
1427                 agaw = gaw + 9 - r;
1428         if (agaw > 64)
1429                 agaw = 64;
1430         return agaw;
1431 }
1432
1433 static int domain_init(struct dmar_domain *domain, int guest_width)
1434 {
1435         struct intel_iommu *iommu;
1436         int adjust_width, agaw;
1437         unsigned long sagaw;
1438
1439         init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1440         spin_lock_init(&domain->iommu_lock);
1441
1442         domain_reserve_special_ranges(domain);
1443
1444         /* calculate AGAW */
1445         iommu = domain_get_iommu(domain);
1446         if (guest_width > cap_mgaw(iommu->cap))
1447                 guest_width = cap_mgaw(iommu->cap);
1448         domain->gaw = guest_width;
1449         adjust_width = guestwidth_to_adjustwidth(guest_width);
1450         agaw = width_to_agaw(adjust_width);
1451         sagaw = cap_sagaw(iommu->cap);
1452         if (!test_bit(agaw, &sagaw)) {
1453                 /* hardware doesn't support it, choose a bigger one */
1454                 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1455                 agaw = find_next_bit(&sagaw, 5, agaw);
1456                 if (agaw >= 5)
1457                         return -ENODEV;
1458         }
1459         domain->agaw = agaw;
1460         INIT_LIST_HEAD(&domain->devices);
1461
1462         if (ecap_coherent(iommu->ecap))
1463                 domain->iommu_coherency = 1;
1464         else
1465                 domain->iommu_coherency = 0;
1466
1467         if (ecap_sc_support(iommu->ecap))
1468                 domain->iommu_snooping = 1;
1469         else
1470                 domain->iommu_snooping = 0;
1471
1472         domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1473         domain->iommu_count = 1;
1474         domain->nid = iommu->node;
1475
1476         /* always allocate the top pgd */
1477         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1478         if (!domain->pgd)
1479                 return -ENOMEM;
1480         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1481         return 0;
1482 }
1483
1484 static void domain_exit(struct dmar_domain *domain)
1485 {
1486         struct dmar_drhd_unit *drhd;
1487         struct intel_iommu *iommu;
1488
1489         /* Domain 0 is reserved, so dont process it */
1490         if (!domain)
1491                 return;
1492
1493         /* Flush any lazy unmaps that may reference this domain */
1494         if (!intel_iommu_strict)
1495                 flush_unmaps_timeout(0);
1496
1497         domain_remove_dev_info(domain);
1498         /* destroy iovas */
1499         put_iova_domain(&domain->iovad);
1500
1501         /* clear ptes */
1502         dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1503
1504         /* free page tables */
1505         dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1506
1507         for_each_active_iommu(iommu, drhd)
1508                 if (test_bit(iommu->seq_id, &domain->iommu_bmp))
1509                         iommu_detach_domain(domain, iommu);
1510
1511         free_domain_mem(domain);
1512 }
1513
1514 static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1515                                  u8 bus, u8 devfn, int translation)
1516 {
1517         struct context_entry *context;
1518         unsigned long flags;
1519         struct intel_iommu *iommu;
1520         struct dma_pte *pgd;
1521         unsigned long num;
1522         unsigned long ndomains;
1523         int id;
1524         int agaw;
1525         struct device_domain_info *info = NULL;
1526
1527         pr_debug("Set context mapping for %02x:%02x.%d\n",
1528                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1529
1530         BUG_ON(!domain->pgd);
1531         BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1532                translation != CONTEXT_TT_MULTI_LEVEL);
1533
1534         iommu = device_to_iommu(segment, bus, devfn);
1535         if (!iommu)
1536                 return -ENODEV;
1537
1538         context = device_to_context_entry(iommu, bus, devfn);
1539         if (!context)
1540                 return -ENOMEM;
1541         spin_lock_irqsave(&iommu->lock, flags);
1542         if (context_present(context)) {
1543                 spin_unlock_irqrestore(&iommu->lock, flags);
1544                 return 0;
1545         }
1546
1547         id = domain->id;
1548         pgd = domain->pgd;
1549
1550         if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1551             domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
1552                 int found = 0;
1553
1554                 /* find an available domain id for this device in iommu */
1555                 ndomains = cap_ndoms(iommu->cap);
1556                 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1557                         if (iommu->domains[num] == domain) {
1558                                 id = num;
1559                                 found = 1;
1560                                 break;
1561                         }
1562                 }
1563
1564                 if (found == 0) {
1565                         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1566                         if (num >= ndomains) {
1567                                 spin_unlock_irqrestore(&iommu->lock, flags);
1568                                 printk(KERN_ERR "IOMMU: no free domain ids\n");
1569                                 return -EFAULT;
1570                         }
1571
1572                         set_bit(num, iommu->domain_ids);
1573                         iommu->domains[num] = domain;
1574                         id = num;
1575                 }
1576
1577                 /* Skip top levels of page tables for
1578                  * iommu which has less agaw than default.
1579                  * Unnecessary for PT mode.
1580                  */
1581                 if (translation != CONTEXT_TT_PASS_THROUGH) {
1582                         for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1583                                 pgd = phys_to_virt(dma_pte_addr(pgd));
1584                                 if (!dma_pte_present(pgd)) {
1585                                         spin_unlock_irqrestore(&iommu->lock, flags);
1586                                         return -ENOMEM;
1587                                 }
1588                         }
1589                 }
1590         }
1591
1592         context_set_domain_id(context, id);
1593
1594         if (translation != CONTEXT_TT_PASS_THROUGH) {
1595                 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1596                 translation = info ? CONTEXT_TT_DEV_IOTLB :
1597                                      CONTEXT_TT_MULTI_LEVEL;
1598         }
1599         /*
1600          * In pass through mode, AW must be programmed to indicate the largest
1601          * AGAW value supported by hardware. And ASR is ignored by hardware.
1602          */
1603         if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
1604                 context_set_address_width(context, iommu->msagaw);
1605         else {
1606                 context_set_address_root(context, virt_to_phys(pgd));
1607                 context_set_address_width(context, iommu->agaw);
1608         }
1609
1610         context_set_translation_type(context, translation);
1611         context_set_fault_enable(context);
1612         context_set_present(context);
1613         domain_flush_cache(domain, context, sizeof(*context));
1614
1615         /*
1616          * It's a non-present to present mapping. If hardware doesn't cache
1617          * non-present entry we only need to flush the write-buffer. If the
1618          * _does_ cache non-present entries, then it does so in the special
1619          * domain #0, which we have to flush:
1620          */
1621         if (cap_caching_mode(iommu->cap)) {
1622                 iommu->flush.flush_context(iommu, 0,
1623                                            (((u16)bus) << 8) | devfn,
1624                                            DMA_CCMD_MASK_NOBIT,
1625                                            DMA_CCMD_DEVICE_INVL);
1626                 iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
1627         } else {
1628                 iommu_flush_write_buffer(iommu);
1629         }
1630         iommu_enable_dev_iotlb(info);
1631         spin_unlock_irqrestore(&iommu->lock, flags);
1632
1633         spin_lock_irqsave(&domain->iommu_lock, flags);
1634         if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {
1635                 domain->iommu_count++;
1636                 if (domain->iommu_count == 1)
1637                         domain->nid = iommu->node;
1638                 domain_update_iommu_cap(domain);
1639         }
1640         spin_unlock_irqrestore(&domain->iommu_lock, flags);
1641         return 0;
1642 }
1643
1644 static int
1645 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1646                         int translation)
1647 {
1648         int ret;
1649         struct pci_dev *tmp, *parent;
1650
1651         ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
1652                                          pdev->bus->number, pdev->devfn,
1653                                          translation);
1654         if (ret)
1655                 return ret;
1656
1657         /* dependent device mapping */
1658         tmp = pci_find_upstream_pcie_bridge(pdev);
1659         if (!tmp)
1660                 return 0;
1661         /* Secondary interface's bus number and devfn 0 */
1662         parent = pdev->bus->self;
1663         while (parent != tmp) {
1664                 ret = domain_context_mapping_one(domain,
1665                                                  pci_domain_nr(parent->bus),
1666                                                  parent->bus->number,
1667                                                  parent->devfn, translation);
1668                 if (ret)
1669                         return ret;
1670                 parent = parent->bus->self;
1671         }
1672         if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
1673                 return domain_context_mapping_one(domain,
1674                                         pci_domain_nr(tmp->subordinate),
1675                                         tmp->subordinate->number, 0,
1676                                         translation);
1677         else /* this is a legacy PCI bridge */
1678                 return domain_context_mapping_one(domain,
1679                                                   pci_domain_nr(tmp->bus),
1680                                                   tmp->bus->number,
1681                                                   tmp->devfn,
1682                                                   translation);
1683 }
1684
1685 static int domain_context_mapped(struct pci_dev *pdev)
1686 {
1687         int ret;
1688         struct pci_dev *tmp, *parent;
1689         struct intel_iommu *iommu;
1690
1691         iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1692                                 pdev->devfn);
1693         if (!iommu)
1694                 return -ENODEV;
1695
1696         ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
1697         if (!ret)
1698                 return ret;
1699         /* dependent device mapping */
1700         tmp = pci_find_upstream_pcie_bridge(pdev);
1701         if (!tmp)
1702                 return ret;
1703         /* Secondary interface's bus number and devfn 0 */
1704         parent = pdev->bus->self;
1705         while (parent != tmp) {
1706                 ret = device_context_mapped(iommu, parent->bus->number,
1707                                             parent->devfn);
1708                 if (!ret)
1709                         return ret;
1710                 parent = parent->bus->self;
1711         }
1712         if (pci_is_pcie(tmp))
1713                 return device_context_mapped(iommu, tmp->subordinate->number,
1714                                              0);
1715         else
1716                 return device_context_mapped(iommu, tmp->bus->number,
1717                                              tmp->devfn);
1718 }
1719
1720 /* Returns a number of VTD pages, but aligned to MM page size */
1721 static inline unsigned long aligned_nrpages(unsigned long host_addr,
1722                                             size_t size)
1723 {
1724         host_addr &= ~PAGE_MASK;
1725         return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1726 }
1727
1728 /* Return largest possible superpage level for a given mapping */
1729 static inline int hardware_largepage_caps(struct dmar_domain *domain,
1730                                           unsigned long iov_pfn,
1731                                           unsigned long phy_pfn,
1732                                           unsigned long pages)
1733 {
1734         int support, level = 1;
1735         unsigned long pfnmerge;
1736
1737         support = domain->iommu_superpage;
1738
1739         /* To use a large page, the virtual *and* physical addresses
1740            must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1741            of them will mean we have to use smaller pages. So just
1742            merge them and check both at once. */
1743         pfnmerge = iov_pfn | phy_pfn;
1744
1745         while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1746                 pages >>= VTD_STRIDE_SHIFT;
1747                 if (!pages)
1748                         break;
1749                 pfnmerge >>= VTD_STRIDE_SHIFT;
1750                 level++;
1751                 support--;
1752         }
1753         return level;
1754 }
1755
1756 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1757                             struct scatterlist *sg, unsigned long phys_pfn,
1758                             unsigned long nr_pages, int prot)
1759 {
1760         struct dma_pte *first_pte = NULL, *pte = NULL;
1761         phys_addr_t uninitialized_var(pteval);
1762         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1763         unsigned long sg_res;
1764         unsigned int largepage_lvl = 0;
1765         unsigned long lvl_pages = 0;
1766
1767         BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1768
1769         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1770                 return -EINVAL;
1771
1772         prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1773
1774         if (sg)
1775                 sg_res = 0;
1776         else {
1777                 sg_res = nr_pages + 1;
1778                 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1779         }
1780
1781         while (nr_pages > 0) {
1782                 uint64_t tmp;
1783
1784                 if (!sg_res) {
1785                         sg_res = aligned_nrpages(sg->offset, sg->length);
1786                         sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1787                         sg->dma_length = sg->length;
1788                         pteval = page_to_phys(sg_page(sg)) | prot;
1789                         phys_pfn = pteval >> VTD_PAGE_SHIFT;
1790                 }
1791
1792                 if (!pte) {
1793                         largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
1794
1795                         first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, largepage_lvl);
1796                         if (!pte)
1797                                 return -ENOMEM;
1798                         /* It is large page*/
1799                         if (largepage_lvl > 1)
1800                                 pteval |= DMA_PTE_LARGE_PAGE;
1801                         else
1802                                 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
1803
1804                 }
1805                 /* We don't need lock here, nobody else
1806                  * touches the iova range
1807                  */
1808                 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
1809                 if (tmp) {
1810                         static int dumps = 5;
1811                         printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1812                                iov_pfn, tmp, (unsigned long long)pteval);
1813                         if (dumps) {
1814                                 dumps--;
1815                                 debug_dma_dump_mappings(NULL);
1816                         }
1817                         WARN_ON(1);
1818                 }
1819
1820                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
1821
1822                 BUG_ON(nr_pages < lvl_pages);
1823                 BUG_ON(sg_res < lvl_pages);
1824
1825                 nr_pages -= lvl_pages;
1826                 iov_pfn += lvl_pages;
1827                 phys_pfn += lvl_pages;
1828                 pteval += lvl_pages * VTD_PAGE_SIZE;
1829                 sg_res -= lvl_pages;
1830
1831                 /* If the next PTE would be the first in a new page, then we
1832                    need to flush the cache on the entries we've just written.
1833                    And then we'll need to recalculate 'pte', so clear it and
1834                    let it get set again in the if (!pte) block above.
1835
1836                    If we're done (!nr_pages) we need to flush the cache too.
1837
1838                    Also if we've been setting superpages, we may need to
1839                    recalculate 'pte' and switch back to smaller pages for the
1840                    end of the mapping, if the trailing size is not enough to
1841                    use another superpage (i.e. sg_res < lvl_pages). */
1842                 pte++;
1843                 if (!nr_pages || first_pte_in_page(pte) ||
1844                     (largepage_lvl > 1 && sg_res < lvl_pages)) {
1845                         domain_flush_cache(domain, first_pte,
1846                                            (void *)pte - (void *)first_pte);
1847                         pte = NULL;
1848                 }
1849
1850                 if (!sg_res && nr_pages)
1851                         sg = sg_next(sg);
1852         }
1853         return 0;
1854 }
1855
1856 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1857                                     struct scatterlist *sg, unsigned long nr_pages,
1858                                     int prot)
1859 {
1860         return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
1861 }
1862
1863 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1864                                      unsigned long phys_pfn, unsigned long nr_pages,
1865                                      int prot)
1866 {
1867         return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
1868 }
1869
1870 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
1871 {
1872         if (!iommu)
1873                 return;
1874
1875         clear_context_table(iommu, bus, devfn);
1876         iommu->flush.flush_context(iommu, 0, 0, 0,
1877                                            DMA_CCMD_GLOBAL_INVL);
1878         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
1879 }
1880
1881 static void domain_remove_dev_info(struct dmar_domain *domain)
1882 {
1883         struct device_domain_info *info;
1884         unsigned long flags;
1885         struct intel_iommu *iommu;
1886
1887         spin_lock_irqsave(&device_domain_lock, flags);
1888         while (!list_empty(&domain->devices)) {
1889                 info = list_entry(domain->devices.next,
1890                         struct device_domain_info, link);
1891                 list_del(&info->link);
1892                 list_del(&info->global);
1893                 if (info->dev)
1894                         info->dev->dev.archdata.iommu = NULL;
1895                 spin_unlock_irqrestore(&device_domain_lock, flags);
1896
1897                 iommu_disable_dev_iotlb(info);
1898                 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
1899                 iommu_detach_dev(iommu, info->bus, info->devfn);
1900                 free_devinfo_mem(info);
1901
1902                 spin_lock_irqsave(&device_domain_lock, flags);
1903         }
1904         spin_unlock_irqrestore(&device_domain_lock, flags);
1905 }
1906
1907 /*
1908  * find_domain
1909  * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1910  */
1911 static struct dmar_domain *
1912 find_domain(struct pci_dev *pdev)
1913 {
1914         struct device_domain_info *info;
1915
1916         /* No lock here, assumes no domain exit in normal case */
1917         info = pdev->dev.archdata.iommu;
1918         if (info)
1919                 return info->domain;
1920         return NULL;
1921 }
1922
1923 /* domain is initialized */
1924 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1925 {
1926         struct dmar_domain *domain, *found = NULL;
1927         struct intel_iommu *iommu;
1928         struct dmar_drhd_unit *drhd;
1929         struct device_domain_info *info, *tmp;
1930         struct pci_dev *dev_tmp;
1931         unsigned long flags;
1932         int bus = 0, devfn = 0;
1933         int segment;
1934         int ret;
1935
1936         domain = find_domain(pdev);
1937         if (domain)
1938                 return domain;
1939
1940         segment = pci_domain_nr(pdev->bus);
1941
1942         dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1943         if (dev_tmp) {
1944                 if (pci_is_pcie(dev_tmp)) {
1945                         bus = dev_tmp->subordinate->number;
1946                         devfn = 0;
1947                 } else {
1948                         bus = dev_tmp->bus->number;
1949                         devfn = dev_tmp->devfn;
1950                 }
1951                 spin_lock_irqsave(&device_domain_lock, flags);
1952                 list_for_each_entry(info, &device_domain_list, global) {
1953                         if (info->segment == segment &&
1954                             info->bus == bus && info->devfn == devfn) {
1955                                 found = info->domain;
1956                                 break;
1957                         }
1958                 }
1959                 spin_unlock_irqrestore(&device_domain_lock, flags);
1960                 /* pcie-pci bridge already has a domain, uses it */
1961                 if (found) {
1962                         domain = found;
1963                         goto found_domain;
1964                 }
1965         }
1966
1967         domain = alloc_domain();
1968         if (!domain)
1969                 goto error;
1970
1971         /* Allocate new domain for the device */
1972         drhd = dmar_find_matched_drhd_unit(pdev);
1973         if (!drhd) {
1974                 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1975                         pci_name(pdev));
1976                 return NULL;
1977         }
1978         iommu = drhd->iommu;
1979
1980         ret = iommu_attach_domain(domain, iommu);
1981         if (ret) {
1982                 free_domain_mem(domain);
1983                 goto error;
1984         }
1985
1986         if (domain_init(domain, gaw)) {
1987                 domain_exit(domain);
1988                 goto error;
1989         }
1990
1991         /* register pcie-to-pci device */
1992         if (dev_tmp) {
1993                 info = alloc_devinfo_mem();
1994                 if (!info) {
1995                         domain_exit(domain);
1996                         goto error;
1997                 }
1998                 info->segment = segment;
1999                 info->bus = bus;
2000                 info->devfn = devfn;
2001                 info->dev = NULL;
2002                 info->domain = domain;
2003                 /* This domain is shared by devices under p2p bridge */
2004                 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
2005
2006                 /* pcie-to-pci bridge already has a domain, uses it */
2007                 found = NULL;
2008                 spin_lock_irqsave(&device_domain_lock, flags);
2009                 list_for_each_entry(tmp, &device_domain_list, global) {
2010                         if (tmp->segment == segment &&
2011                             tmp->bus == bus && tmp->devfn == devfn) {
2012                                 found = tmp->domain;
2013                                 break;
2014                         }
2015                 }
2016                 if (found) {
2017                         spin_unlock_irqrestore(&device_domain_lock, flags);
2018                         free_devinfo_mem(info);
2019                         domain_exit(domain);
2020                         domain = found;
2021                 } else {
2022                         list_add(&info->link, &domain->devices);
2023                         list_add(&info->global, &device_domain_list);
2024                         spin_unlock_irqrestore(&device_domain_lock, flags);
2025                 }
2026         }
2027
2028 found_domain:
2029         info = alloc_devinfo_mem();
2030         if (!info)
2031                 goto error;
2032         info->segment = segment;
2033         info->bus = pdev->bus->number;
2034         info->devfn = pdev->devfn;
2035         info->dev = pdev;
2036         info->domain = domain;
2037         spin_lock_irqsave(&device_domain_lock, flags);
2038         /* somebody is fast */
2039         found = find_domain(pdev);
2040         if (found != NULL) {
2041                 spin_unlock_irqrestore(&device_domain_lock, flags);
2042                 if (found != domain) {
2043                         domain_exit(domain);
2044                         domain = found;
2045                 }
2046                 free_devinfo_mem(info);
2047                 return domain;
2048         }
2049         list_add(&info->link, &domain->devices);
2050         list_add(&info->global, &device_domain_list);
2051         pdev->dev.archdata.iommu = info;
2052         spin_unlock_irqrestore(&device_domain_lock, flags);
2053         return domain;
2054 error:
2055         /* recheck it here, maybe others set it */
2056         return find_domain(pdev);
2057 }
2058
2059 static int iommu_identity_mapping;
2060 #define IDENTMAP_ALL            1
2061 #define IDENTMAP_GFX            2
2062 #define IDENTMAP_AZALIA         4
2063
2064 static int iommu_domain_identity_map(struct dmar_domain *domain,
2065                                      unsigned long long start,
2066                                      unsigned long long end)
2067 {
2068         unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2069         unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2070
2071         if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2072                           dma_to_mm_pfn(last_vpfn))) {
2073                 printk(KERN_ERR "IOMMU: reserve iova failed\n");
2074                 return -ENOMEM;
2075         }
2076
2077         pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2078                  start, end, domain->id);
2079         /*
2080          * RMRR range might have overlap with physical memory range,
2081          * clear it first
2082          */
2083         dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2084
2085         return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2086                                   last_vpfn - first_vpfn + 1,
2087                                   DMA_PTE_READ|DMA_PTE_WRITE);
2088 }
2089
2090 static int iommu_prepare_identity_map(struct pci_dev *pdev,
2091                                       unsigned long long start,
2092                                       unsigned long long end)
2093 {
2094         struct dmar_domain *domain;
2095         int ret;
2096
2097         domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2098         if (!domain)
2099                 return -ENOMEM;
2100
2101         /* For _hardware_ passthrough, don't bother. But for software
2102            passthrough, we do it anyway -- it may indicate a memory
2103            range which is reserved in E820, so which didn't get set
2104            up to start with in si_domain */
2105         if (domain == si_domain && hw_pass_through) {
2106                 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2107                        pci_name(pdev), start, end);
2108                 return 0;
2109         }
2110
2111         printk(KERN_INFO
2112                "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2113                pci_name(pdev), start, end);
2114
2115         if (end < start) {
2116                 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2117                         "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2118                         dmi_get_system_info(DMI_BIOS_VENDOR),
2119                         dmi_get_system_info(DMI_BIOS_VERSION),
2120                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2121                 ret = -EIO;
2122                 goto error;
2123         }
2124
2125         if (end >> agaw_to_width(domain->agaw)) {
2126                 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2127                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2128                      agaw_to_width(domain->agaw),
2129                      dmi_get_system_info(DMI_BIOS_VENDOR),
2130                      dmi_get_system_info(DMI_BIOS_VERSION),
2131                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2132                 ret = -EIO;
2133                 goto error;
2134         }
2135
2136         ret = iommu_domain_identity_map(domain, start, end);
2137         if (ret)
2138                 goto error;
2139
2140         /* context entry init */
2141         ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
2142         if (ret)
2143                 goto error;
2144
2145         return 0;
2146
2147  error:
2148         domain_exit(domain);
2149         return ret;
2150 }
2151
2152 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2153         struct pci_dev *pdev)
2154 {
2155         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2156                 return 0;
2157         return iommu_prepare_identity_map(pdev, rmrr->base_address,
2158                 rmrr->end_address);
2159 }
2160
2161 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2162 static inline void iommu_prepare_isa(void)
2163 {
2164         struct pci_dev *pdev;
2165         int ret;
2166
2167         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2168         if (!pdev)
2169                 return;
2170
2171         printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
2172         ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024 - 1);
2173
2174         if (ret)
2175                 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2176                        "floppy might not work\n");
2177
2178 }
2179 #else
2180 static inline void iommu_prepare_isa(void)
2181 {
2182         return;
2183 }
2184 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2185
2186 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2187
2188 static int __init si_domain_work_fn(unsigned long start_pfn,
2189                                     unsigned long end_pfn, void *datax)
2190 {
2191         int *ret = datax;
2192
2193         *ret = iommu_domain_identity_map(si_domain,
2194                                          (uint64_t)start_pfn << PAGE_SHIFT,
2195                                          (uint64_t)end_pfn << PAGE_SHIFT);
2196         return *ret;
2197
2198 }
2199
2200 static int __init si_domain_init(int hw)
2201 {
2202         struct dmar_drhd_unit *drhd;
2203         struct intel_iommu *iommu;
2204         int nid, ret = 0;
2205
2206         si_domain = alloc_domain();
2207         if (!si_domain)
2208                 return -EFAULT;
2209
2210         pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
2211
2212         for_each_active_iommu(iommu, drhd) {
2213                 ret = iommu_attach_domain(si_domain, iommu);
2214                 if (ret) {
2215                         domain_exit(si_domain);
2216                         return -EFAULT;
2217                 }
2218         }
2219
2220         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2221                 domain_exit(si_domain);
2222                 return -EFAULT;
2223         }
2224
2225         si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2226
2227         if (hw)
2228                 return 0;
2229
2230         for_each_online_node(nid) {
2231                 work_with_active_regions(nid, si_domain_work_fn, &ret);
2232                 if (ret)
2233                         return ret;
2234         }
2235
2236         return 0;
2237 }
2238
2239 static void domain_remove_one_dev_info(struct dmar_domain *domain,
2240                                           struct pci_dev *pdev);
2241 static int identity_mapping(struct pci_dev *pdev)
2242 {
2243         struct device_domain_info *info;
2244
2245         if (likely(!iommu_identity_mapping))
2246                 return 0;
2247
2248         info = pdev->dev.archdata.iommu;
2249         if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2250                 return (info->domain == si_domain);
2251
2252         return 0;
2253 }
2254
2255 static int domain_add_dev_info(struct dmar_domain *domain,
2256                                struct pci_dev *pdev,
2257                                int translation)
2258 {
2259         struct device_domain_info *info;
2260         unsigned long flags;
2261         int ret;
2262
2263         info = alloc_devinfo_mem();
2264         if (!info)
2265                 return -ENOMEM;
2266
2267         ret = domain_context_mapping(domain, pdev, translation);
2268         if (ret) {
2269                 free_devinfo_mem(info);
2270                 return ret;
2271         }
2272
2273         info->segment = pci_domain_nr(pdev->bus);
2274         info->bus = pdev->bus->number;
2275         info->devfn = pdev->devfn;
2276         info->dev = pdev;
2277         info->domain = domain;
2278
2279         spin_lock_irqsave(&device_domain_lock, flags);
2280         list_add(&info->link, &domain->devices);
2281         list_add(&info->global, &device_domain_list);
2282         pdev->dev.archdata.iommu = info;
2283         spin_unlock_irqrestore(&device_domain_lock, flags);
2284
2285         return 0;
2286 }
2287
2288 static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
2289 {
2290         if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2291                 return 1;
2292
2293         if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2294                 return 1;
2295
2296         if (!(iommu_identity_mapping & IDENTMAP_ALL))
2297                 return 0;
2298
2299         /*
2300          * We want to start off with all devices in the 1:1 domain, and
2301          * take them out later if we find they can't access all of memory.
2302          *
2303          * However, we can't do this for PCI devices behind bridges,
2304          * because all PCI devices behind the same bridge will end up
2305          * with the same source-id on their transactions.
2306          *
2307          * Practically speaking, we can't change things around for these
2308          * devices at run-time, because we can't be sure there'll be no
2309          * DMA transactions in flight for any of their siblings.
2310          *
2311          * So PCI devices (unless they're on the root bus) as well as
2312          * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2313          * the 1:1 domain, just in _case_ one of their siblings turns out
2314          * not to be able to map all of memory.
2315          */
2316         if (!pci_is_pcie(pdev)) {
2317                 if (!pci_is_root_bus(pdev->bus))
2318                         return 0;
2319                 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2320                         return 0;
2321         } else if (pdev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
2322                 return 0;
2323
2324         /*
2325          * At boot time, we don't yet know if devices will be 64-bit capable.
2326          * Assume that they will -- if they turn out not to be, then we can
2327          * take them out of the 1:1 domain later.
2328          */
2329         if (!startup) {
2330                 /*
2331                  * If the device's dma_mask is less than the system's memory
2332                  * size then this is not a candidate for identity mapping.
2333                  */
2334                 u64 dma_mask = pdev->dma_mask;
2335
2336                 if (pdev->dev.coherent_dma_mask &&
2337                     pdev->dev.coherent_dma_mask < dma_mask)
2338                         dma_mask = pdev->dev.coherent_dma_mask;
2339
2340                 return dma_mask >= dma_get_required_mask(&pdev->dev);
2341         }
2342
2343         return 1;
2344 }
2345
2346 static int __init iommu_prepare_static_identity_mapping(int hw)
2347 {
2348         struct pci_dev *pdev = NULL;
2349         int ret;
2350
2351         ret = si_domain_init(hw);
2352         if (ret)
2353                 return -EFAULT;
2354
2355         for_each_pci_dev(pdev) {
2356                 /* Skip Host/PCI Bridge devices */
2357                 if (IS_BRIDGE_HOST_DEVICE(pdev))
2358                         continue;
2359                 if (iommu_should_identity_map(pdev, 1)) {
2360                         printk(KERN_INFO "IOMMU: %s identity mapping for device %s\n",
2361                                hw ? "hardware" : "software", pci_name(pdev));
2362
2363                         ret = domain_add_dev_info(si_domain, pdev,
2364                                                      hw ? CONTEXT_TT_PASS_THROUGH :
2365                                                      CONTEXT_TT_MULTI_LEVEL);
2366                         if (ret)
2367                                 return ret;
2368                 }
2369         }
2370
2371         return 0;
2372 }
2373
2374 static int __init init_dmars(void)
2375 {
2376         struct dmar_drhd_unit *drhd;
2377         struct dmar_rmrr_unit *rmrr;
2378         struct pci_dev *pdev;
2379         struct intel_iommu *iommu;
2380         int i, ret;
2381
2382         /*
2383          * for each drhd
2384          *    allocate root
2385          *    initialize and program root entry to not present
2386          * endfor
2387          */
2388         for_each_drhd_unit(drhd) {
2389                 g_num_of_iommus++;
2390                 /*
2391                  * lock not needed as this is only incremented in the single
2392                  * threaded kernel __init code path all other access are read
2393                  * only
2394                  */
2395         }
2396
2397         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2398                         GFP_KERNEL);
2399         if (!g_iommus) {
2400                 printk(KERN_ERR "Allocating global iommu array failed\n");
2401                 ret = -ENOMEM;
2402                 goto error;
2403         }
2404
2405         deferred_flush = kzalloc(g_num_of_iommus *
2406                 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2407         if (!deferred_flush) {
2408                 ret = -ENOMEM;
2409                 goto error;
2410         }
2411
2412         for_each_drhd_unit(drhd) {
2413                 if (drhd->ignored)
2414                         continue;
2415
2416                 iommu = drhd->iommu;
2417                 g_iommus[iommu->seq_id] = iommu;
2418
2419                 ret = iommu_init_domains(iommu);
2420                 if (ret)
2421                         goto error;
2422
2423                 /*
2424                  * TBD:
2425                  * we could share the same root & context tables
2426                  * among all IOMMU's. Need to Split it later.
2427                  */
2428                 ret = iommu_alloc_root_entry(iommu);
2429                 if (ret) {
2430                         printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2431                         goto error;
2432                 }
2433                 if (!ecap_pass_through(iommu->ecap))
2434                         hw_pass_through = 0;
2435         }
2436
2437         /*
2438          * Start from the sane iommu hardware state.
2439          */
2440         for_each_drhd_unit(drhd) {
2441                 if (drhd->ignored)
2442                         continue;
2443
2444                 iommu = drhd->iommu;
2445
2446                 /*
2447                  * If the queued invalidation is already initialized by us
2448                  * (for example, while enabling interrupt-remapping) then
2449                  * we got the things already rolling from a sane state.
2450                  */
2451                 if (iommu->qi)
2452                         continue;
2453
2454                 /*
2455                  * Clear any previous faults.
2456                  */
2457                 dmar_fault(-1, iommu);
2458                 /*
2459                  * Disable queued invalidation if supported and already enabled
2460                  * before OS handover.
2461                  */
2462                 dmar_disable_qi(iommu);
2463         }
2464
2465         for_each_drhd_unit(drhd) {
2466                 if (drhd->ignored)
2467                         continue;
2468
2469                 iommu = drhd->iommu;
2470
2471                 if (dmar_enable_qi(iommu)) {
2472                         /*
2473                          * Queued Invalidate not enabled, use Register Based
2474                          * Invalidate
2475                          */
2476                         iommu->flush.flush_context = __iommu_flush_context;
2477                         iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2478                         printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
2479                                "invalidation\n",
2480                                 iommu->seq_id,
2481                                (unsigned long long)drhd->reg_base_addr);
2482                 } else {
2483                         iommu->flush.flush_context = qi_flush_context;
2484                         iommu->flush.flush_iotlb = qi_flush_iotlb;
2485                         printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
2486                                "invalidation\n",
2487                                 iommu->seq_id,
2488                                (unsigned long long)drhd->reg_base_addr);
2489                 }
2490         }
2491
2492         if (iommu_pass_through)
2493                 iommu_identity_mapping |= IDENTMAP_ALL;
2494
2495 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
2496         iommu_identity_mapping |= IDENTMAP_GFX;
2497 #endif
2498
2499         check_tylersburg_isoch();
2500
2501         /*
2502          * If pass through is not set or not enabled, setup context entries for
2503          * identity mappings for rmrr, gfx, and isa and may fall back to static
2504          * identity mapping if iommu_identity_mapping is set.
2505          */
2506         if (iommu_identity_mapping) {
2507                 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
2508                 if (ret) {
2509                         printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2510                         goto error;
2511                 }
2512         }
2513         /*
2514          * For each rmrr
2515          *   for each dev attached to rmrr
2516          *   do
2517          *     locate drhd for dev, alloc domain for dev
2518          *     allocate free domain
2519          *     allocate page table entries for rmrr
2520          *     if context not allocated for bus
2521          *           allocate and init context
2522          *           set present in root table for this bus
2523          *     init context with domain, translation etc
2524          *    endfor
2525          * endfor
2526          */
2527         printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2528         for_each_rmrr_units(rmrr) {
2529                 for (i = 0; i < rmrr->devices_cnt; i++) {
2530                         pdev = rmrr->devices[i];
2531                         /*
2532                          * some BIOS lists non-exist devices in DMAR
2533                          * table.
2534                          */
2535                         if (!pdev)
2536                                 continue;
2537                         ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2538                         if (ret)
2539                                 printk(KERN_ERR
2540                                        "IOMMU: mapping reserved region failed\n");
2541                 }
2542         }
2543
2544         iommu_prepare_isa();
2545
2546         /*
2547          * for each drhd
2548          *   enable fault log
2549          *   global invalidate context cache
2550          *   global invalidate iotlb
2551          *   enable translation
2552          */
2553         for_each_drhd_unit(drhd) {
2554                 if (drhd->ignored) {
2555                         /*
2556                          * we always have to disable PMRs or DMA may fail on
2557                          * this device
2558                          */
2559                         if (force_on)
2560                                 iommu_disable_protect_mem_regions(drhd->iommu);
2561                         continue;
2562                 }
2563                 iommu = drhd->iommu;
2564
2565                 iommu_flush_write_buffer(iommu);
2566
2567                 ret = dmar_set_interrupt(iommu);
2568                 if (ret)
2569                         goto error;
2570
2571                 iommu_set_root_entry(iommu);
2572
2573                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
2574                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2575
2576                 ret = iommu_enable_translation(iommu);
2577                 if (ret)
2578                         goto error;
2579
2580                 iommu_disable_protect_mem_regions(iommu);
2581         }
2582
2583         return 0;
2584 error:
2585         for_each_drhd_unit(drhd) {
2586                 if (drhd->ignored)
2587                         continue;
2588                 iommu = drhd->iommu;
2589                 free_iommu(iommu);
2590         }
2591         kfree(g_iommus);
2592         return ret;
2593 }
2594
2595 /* This takes a number of _MM_ pages, not VTD pages */
2596 static struct iova *intel_alloc_iova(struct device *dev,
2597                                      struct dmar_domain *domain,
2598                                      unsigned long nrpages, uint64_t dma_mask)
2599 {
2600         struct pci_dev *pdev = to_pci_dev(dev);
2601         struct iova *iova = NULL;
2602
2603         /* Restrict dma_mask to the width that the iommu can handle */
2604         dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2605
2606         if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
2607                 /*
2608                  * First try to allocate an io virtual address in
2609                  * DMA_BIT_MASK(32) and if that fails then try allocating
2610                  * from higher range
2611                  */
2612                 iova = alloc_iova(&domain->iovad, nrpages,
2613                                   IOVA_PFN(DMA_BIT_MASK(32)), 1);
2614                 if (iova)
2615                         return iova;
2616         }
2617         iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2618         if (unlikely(!iova)) {
2619                 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2620                        nrpages, pci_name(pdev));
2621                 return NULL;
2622         }
2623
2624         return iova;
2625 }
2626
2627 static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
2628 {
2629         struct dmar_domain *domain;
2630         int ret;
2631
2632         domain = get_domain_for_dev(pdev,
2633                         DEFAULT_DOMAIN_ADDRESS_WIDTH);
2634         if (!domain) {
2635                 printk(KERN_ERR
2636                         "Allocating domain for %s failed", pci_name(pdev));
2637                 return NULL;
2638         }
2639
2640         /* make sure context mapping is ok */
2641         if (unlikely(!domain_context_mapped(pdev))) {
2642                 ret = domain_context_mapping(domain, pdev,
2643                                              CONTEXT_TT_MULTI_LEVEL);
2644                 if (ret) {
2645                         printk(KERN_ERR
2646                                 "Domain context map for %s failed",
2647                                 pci_name(pdev));
2648                         return NULL;
2649                 }
2650         }
2651
2652         return domain;
2653 }
2654
2655 static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
2656 {
2657         struct device_domain_info *info;
2658
2659         /* No lock here, assumes no domain exit in normal case */
2660         info = dev->dev.archdata.iommu;
2661         if (likely(info))
2662                 return info->domain;
2663
2664         return __get_valid_domain_for_dev(dev);
2665 }
2666
2667 static int iommu_dummy(struct pci_dev *pdev)
2668 {
2669         return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2670 }
2671
2672 /* Check if the pdev needs to go through non-identity map and unmap process.*/
2673 static int iommu_no_mapping(struct device *dev)
2674 {
2675         struct pci_dev *pdev;
2676         int found;
2677
2678         if (unlikely(dev->bus != &pci_bus_type))
2679                 return 1;
2680
2681         pdev = to_pci_dev(dev);
2682         if (iommu_dummy(pdev))
2683                 return 1;
2684
2685         if (!iommu_identity_mapping)
2686                 return 0;
2687
2688         found = identity_mapping(pdev);
2689         if (found) {
2690                 if (iommu_should_identity_map(pdev, 0))
2691                         return 1;
2692                 else {
2693                         /*
2694                          * 32 bit DMA is removed from si_domain and fall back
2695                          * to non-identity mapping.
2696                          */
2697                         domain_remove_one_dev_info(si_domain, pdev);
2698                         printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2699                                pci_name(pdev));
2700                         return 0;
2701                 }
2702         } else {
2703                 /*
2704                  * In case of a detached 64 bit DMA device from vm, the device
2705                  * is put into si_domain for identity mapping.
2706                  */
2707                 if (iommu_should_identity_map(pdev, 0)) {
2708                         int ret;
2709                         ret = domain_add_dev_info(si_domain, pdev,
2710                                                   hw_pass_through ?
2711                                                   CONTEXT_TT_PASS_THROUGH :
2712                                                   CONTEXT_TT_MULTI_LEVEL);
2713                         if (!ret) {
2714                                 printk(KERN_INFO "64bit %s uses identity mapping\n",
2715                                        pci_name(pdev));
2716                                 return 1;
2717                         }
2718                 }
2719         }
2720
2721         return 0;
2722 }
2723
2724 static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2725                                      size_t size, int dir, u64 dma_mask)
2726 {
2727         struct pci_dev *pdev = to_pci_dev(hwdev);
2728         struct dmar_domain *domain;
2729         phys_addr_t start_paddr;
2730         struct iova *iova;
2731         int prot = 0;
2732         int ret;
2733         struct intel_iommu *iommu;
2734         unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
2735
2736         BUG_ON(dir == DMA_NONE);
2737
2738         if (iommu_no_mapping(hwdev))
2739                 return paddr;
2740
2741         domain = get_valid_domain_for_dev(pdev);
2742         if (!domain)
2743                 return 0;
2744
2745         iommu = domain_get_iommu(domain);
2746         size = aligned_nrpages(paddr, size);
2747
2748         iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size), dma_mask);
2749         if (!iova)
2750                 goto error;
2751
2752         /*
2753          * Check if DMAR supports zero-length reads on write only
2754          * mappings..
2755          */
2756         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2757                         !cap_zlr(iommu->cap))
2758                 prot |= DMA_PTE_READ;
2759         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2760                 prot |= DMA_PTE_WRITE;
2761         /*
2762          * paddr - (paddr + size) might be partial page, we should map the whole
2763          * page.  Note: if two part of one page are separately mapped, we
2764          * might have two guest_addr mapping to the same host paddr, but this
2765          * is not a big problem
2766          */
2767         ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
2768                                  mm_to_dma_pfn(paddr_pfn), size, prot);
2769         if (ret)
2770                 goto error;
2771
2772         /* it's a non-present to present mapping. Only flush if caching mode */
2773         if (cap_caching_mode(iommu->cap))
2774                 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 1);
2775         else
2776                 iommu_flush_write_buffer(iommu);
2777
2778         start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2779         start_paddr += paddr & ~PAGE_MASK;
2780         return start_paddr;
2781
2782 error:
2783         if (iova)
2784                 __free_iova(&domain->iovad, iova);
2785         printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
2786                 pci_name(pdev), size, (unsigned long long)paddr, dir);
2787         return 0;
2788 }
2789
2790 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2791                                  unsigned long offset, size_t size,
2792                                  enum dma_data_direction dir,
2793                                  struct dma_attrs *attrs)
2794 {
2795         return __intel_map_single(dev, page_to_phys(page) + offset, size,
2796                                   dir, to_pci_dev(dev)->dma_mask);
2797 }
2798
2799 static void flush_unmaps(void)
2800 {
2801         int i, j;
2802
2803         timer_on = 0;
2804
2805         /* just flush them all */
2806         for (i = 0; i < g_num_of_iommus; i++) {
2807                 struct intel_iommu *iommu = g_iommus[i];
2808                 if (!iommu)
2809                         continue;
2810
2811                 if (!deferred_flush[i].next)
2812                         continue;
2813
2814                 /* In caching mode, global flushes turn emulation expensive */
2815                 if (!cap_caching_mode(iommu->cap))
2816                         iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2817                                          DMA_TLB_GLOBAL_FLUSH);
2818                 for (j = 0; j < deferred_flush[i].next; j++) {
2819                         unsigned long mask;
2820                         struct iova *iova = deferred_flush[i].iova[j];
2821                         struct dmar_domain *domain = deferred_flush[i].domain[j];
2822
2823                         /* On real hardware multiple invalidations are expensive */
2824                         if (cap_caching_mode(iommu->cap))
2825                                 iommu_flush_iotlb_psi(iommu, domain->id,
2826                                 iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 0);
2827                         else {
2828                                 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
2829                                 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
2830                                                 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
2831                         }
2832                         __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
2833                 }
2834                 deferred_flush[i].next = 0;
2835         }
2836
2837         list_size = 0;
2838 }
2839
2840 static void flush_unmaps_timeout(unsigned long data)
2841 {
2842         unsigned long flags;
2843
2844         spin_lock_irqsave(&async_umap_flush_lock, flags);
2845         flush_unmaps();
2846         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2847 }
2848
2849 static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2850 {
2851         unsigned long flags;
2852         int next, iommu_id;
2853         struct intel_iommu *iommu;
2854
2855         spin_lock_irqsave(&async_umap_flush_lock, flags);
2856         if (list_size == HIGH_WATER_MARK)
2857                 flush_unmaps();
2858
2859         iommu = domain_get_iommu(dom);
2860         iommu_id = iommu->seq_id;
2861
2862         next = deferred_flush[iommu_id].next;
2863         deferred_flush[iommu_id].domain[next] = dom;
2864         deferred_flush[iommu_id].iova[next] = iova;
2865         deferred_flush[iommu_id].next++;
2866
2867         if (!timer_on) {
2868                 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2869                 timer_on = 1;
2870         }
2871         list_size++;
2872         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2873 }
2874
2875 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2876                              size_t size, enum dma_data_direction dir,
2877                              struct dma_attrs *attrs)
2878 {
2879         struct pci_dev *pdev = to_pci_dev(dev);
2880         struct dmar_domain *domain;
2881         unsigned long start_pfn, last_pfn;
2882         struct iova *iova;
2883         struct intel_iommu *iommu;
2884
2885         if (iommu_no_mapping(dev))
2886                 return;
2887
2888         domain = find_domain(pdev);
2889         BUG_ON(!domain);
2890
2891         iommu = domain_get_iommu(domain);
2892
2893         iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
2894         if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
2895                       (unsigned long long)dev_addr))
2896                 return;
2897
2898         start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2899         last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2900
2901         pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2902                  pci_name(pdev), start_pfn, last_pfn);
2903
2904         /*  clear the whole page */
2905         dma_pte_clear_range(domain, start_pfn, last_pfn);
2906
2907         /* free page tables */
2908         dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2909
2910         if (intel_iommu_strict) {
2911                 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2912                                       last_pfn - start_pfn + 1, 0);
2913                 /* free iova */
2914                 __free_iova(&domain->iovad, iova);
2915         } else {
2916                 add_unmap(domain, iova);
2917                 /*
2918                  * queue up the release of the unmap to save the 1/6th of the
2919                  * cpu used up by the iotlb flush operation...
2920                  */
2921         }
2922 }
2923
2924 static void *intel_alloc_coherent(struct device *hwdev, size_t size,
2925                                   dma_addr_t *dma_handle, gfp_t flags)
2926 {
2927         void *vaddr;
2928         int order;
2929
2930         size = PAGE_ALIGN(size);
2931         order = get_order(size);
2932
2933         if (!iommu_no_mapping(hwdev))
2934                 flags &= ~(GFP_DMA | GFP_DMA32);
2935         else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
2936                 if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
2937                         flags |= GFP_DMA;
2938                 else
2939                         flags |= GFP_DMA32;
2940         }
2941
2942         vaddr = (void *)__get_free_pages(flags, order);
2943         if (!vaddr)
2944                 return NULL;
2945         memset(vaddr, 0, size);
2946
2947         *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2948                                          DMA_BIDIRECTIONAL,
2949                                          hwdev->coherent_dma_mask);
2950         if (*dma_handle)
2951                 return vaddr;
2952         free_pages((unsigned long)vaddr, order);
2953         return NULL;
2954 }
2955
2956 static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2957                                 dma_addr_t dma_handle)
2958 {
2959         int order;
2960
2961         size = PAGE_ALIGN(size);
2962         order = get_order(size);
2963
2964         intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
2965         free_pages((unsigned long)vaddr, order);
2966 }
2967
2968 static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2969                            int nelems, enum dma_data_direction dir,
2970                            struct dma_attrs *attrs)
2971 {
2972         struct pci_dev *pdev = to_pci_dev(hwdev);
2973         struct dmar_domain *domain;
2974         unsigned long start_pfn, last_pfn;
2975         struct iova *iova;
2976         struct intel_iommu *iommu;
2977
2978         if (iommu_no_mapping(hwdev))
2979                 return;
2980
2981         domain = find_domain(pdev);
2982         BUG_ON(!domain);
2983
2984         iommu = domain_get_iommu(domain);
2985
2986         iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
2987         if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
2988                       (unsigned long long)sglist[0].dma_address))
2989                 return;
2990
2991         start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2992         last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2993
2994         /*  clear the whole page */
2995         dma_pte_clear_range(domain, start_pfn, last_pfn);
2996
2997         /* free page tables */
2998         dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2999
3000         if (intel_iommu_strict) {
3001                 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
3002                                       last_pfn - start_pfn + 1, 0);
3003                 /* free iova */
3004                 __free_iova(&domain->iovad, iova);
3005         } else {
3006                 add_unmap(domain, iova);
3007                 /*
3008                  * queue up the release of the unmap to save the 1/6th of the
3009                  * cpu used up by the iotlb flush operation...
3010                  */
3011         }
3012 }
3013
3014 static int intel_nontranslate_map_sg(struct device *hddev,
3015         struct scatterlist *sglist, int nelems, int dir)
3016 {
3017         int i;
3018         struct scatterlist *sg;
3019
3020         for_each_sg(sglist, sg, nelems, i) {
3021                 BUG_ON(!sg_page(sg));
3022                 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
3023                 sg->dma_length = sg->length;
3024         }
3025         return nelems;
3026 }
3027
3028 static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
3029                         enum dma_data_direction dir, struct dma_attrs *attrs)
3030 {
3031         int i;
3032         struct pci_dev *pdev = to_pci_dev(hwdev);
3033         struct dmar_domain *domain;
3034         size_t size = 0;
3035         int prot = 0;
3036         struct iova *iova = NULL;
3037         int ret;
3038         struct scatterlist *sg;
3039         unsigned long start_vpfn;
3040         struct intel_iommu *iommu;
3041
3042         BUG_ON(dir == DMA_NONE);
3043         if (iommu_no_mapping(hwdev))
3044                 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
3045
3046         domain = get_valid_domain_for_dev(pdev);
3047         if (!domain)
3048                 return 0;
3049
3050         iommu = domain_get_iommu(domain);
3051
3052         for_each_sg(sglist, sg, nelems, i)
3053                 size += aligned_nrpages(sg->offset, sg->length);
3054
3055         iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
3056                                 pdev->dma_mask);
3057         if (!iova) {
3058                 sglist->dma_length = 0;
3059                 return 0;
3060         }
3061
3062         /*
3063          * Check if DMAR supports zero-length reads on write only
3064          * mappings..
3065          */
3066         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3067                         !cap_zlr(iommu->cap))
3068                 prot |= DMA_PTE_READ;
3069         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3070                 prot |= DMA_PTE_WRITE;
3071
3072         start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
3073
3074         ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3075         if (unlikely(ret)) {
3076                 /*  clear the page */
3077                 dma_pte_clear_range(domain, start_vpfn,
3078                                     start_vpfn + size - 1);
3079                 /* free page tables */
3080                 dma_pte_free_pagetable(domain, start_vpfn,
3081                                        start_vpfn + size - 1);
3082                 /* free iova */
3083                 __free_iova(&domain->iovad, iova);
3084                 return 0;
3085         }
3086
3087         /* it's a non-present to present mapping. Only flush if caching mode */
3088         if (cap_caching_mode(iommu->cap))
3089                 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 1);
3090         else
3091                 iommu_flush_write_buffer(iommu);
3092
3093         return nelems;
3094 }
3095
3096 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3097 {
3098         return !dma_addr;
3099 }
3100
3101 struct dma_map_ops intel_dma_ops = {
3102         .alloc_coherent = intel_alloc_coherent,
3103         .free_coherent = intel_free_coherent,
3104         .map_sg = intel_map_sg,
3105         .unmap_sg = intel_unmap_sg,
3106         .map_page = intel_map_page,
3107         .unmap_page = intel_unmap_page,
3108         .mapping_error = intel_mapping_error,
3109 };
3110
3111 static inline int iommu_domain_cache_init(void)
3112 {
3113         int ret = 0;
3114
3115         iommu_domain_cache = kmem_cache_create("iommu_domain",
3116                                          sizeof(struct dmar_domain),
3117                                          0,
3118                                          SLAB_HWCACHE_ALIGN,
3119
3120                                          NULL);
3121         if (!iommu_domain_cache) {
3122                 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
3123                 ret = -ENOMEM;
3124         }
3125
3126         return ret;
3127 }
3128
3129 static inline int iommu_devinfo_cache_init(void)
3130 {
3131         int ret = 0;
3132
3133         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3134                                          sizeof(struct device_domain_info),
3135                                          0,
3136                                          SLAB_HWCACHE_ALIGN,
3137                                          NULL);
3138         if (!iommu_devinfo_cache) {
3139                 printk(KERN_ERR "Couldn't create devinfo cache\n");
3140                 ret = -ENOMEM;
3141         }
3142
3143         return ret;
3144 }
3145
3146 static inline int iommu_iova_cache_init(void)
3147 {
3148         int ret = 0;
3149
3150         iommu_iova_cache = kmem_cache_create("iommu_iova",
3151                                          sizeof(struct iova),
3152                                          0,
3153                                          SLAB_HWCACHE_ALIGN,
3154                                          NULL);
3155         if (!iommu_iova_cache) {
3156                 printk(KERN_ERR "Couldn't create iova cache\n");
3157                 ret = -ENOMEM;
3158         }
3159
3160         return ret;
3161 }
3162
3163 static int __init iommu_init_mempool(void)
3164 {
3165         int ret;
3166         ret = iommu_iova_cache_init();
3167         if (ret)
3168                 return ret;
3169
3170         ret = iommu_domain_cache_init();
3171         if (ret)
3172                 goto domain_error;
3173
3174         ret = iommu_devinfo_cache_init();
3175         if (!ret)
3176                 return ret;
3177
3178         kmem_cache_destroy(iommu_domain_cache);
3179 domain_error:
3180         kmem_cache_destroy(iommu_iova_cache);
3181
3182         return -ENOMEM;
3183 }
3184
3185 static void __init iommu_exit_mempool(void)
3186 {
3187         kmem_cache_destroy(iommu_devinfo_cache);
3188         kmem_cache_destroy(iommu_domain_cache);
3189         kmem_cache_destroy(iommu_iova_cache);
3190
3191 }
3192
3193 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3194 {
3195         struct dmar_drhd_unit *drhd;
3196         u32 vtbar;
3197         int rc;
3198
3199         /* We know that this device on this chipset has its own IOMMU.
3200          * If we find it under a different IOMMU, then the BIOS is lying
3201          * to us. Hope that the IOMMU for this device is actually
3202          * disabled, and it needs no translation...
3203          */
3204         rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3205         if (rc) {
3206                 /* "can't" happen */
3207                 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3208                 return;
3209         }
3210         vtbar &= 0xffff0000;
3211
3212         /* we know that the this iommu should be at offset 0xa000 from vtbar */
3213         drhd = dmar_find_matched_drhd_unit(pdev);
3214         if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3215                             TAINT_FIRMWARE_WORKAROUND,
3216                             "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3217                 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3218 }
3219 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3220
3221 static void __init init_no_remapping_devices(void)
3222 {
3223         struct dmar_drhd_unit *drhd;
3224
3225         for_each_drhd_unit(drhd) {
3226                 if (!drhd->include_all) {
3227                         int i;
3228                         for (i = 0; i < drhd->devices_cnt; i++)
3229                                 if (drhd->devices[i] != NULL)
3230                                         break;
3231                         /* ignore DMAR unit if no pci devices exist */
3232                         if (i == drhd->devices_cnt)
3233                                 drhd->ignored = 1;
3234                 }
3235         }
3236
3237         for_each_drhd_unit(drhd) {
3238                 int i;
3239                 if (drhd->ignored || drhd->include_all)
3240                         continue;
3241
3242                 for (i = 0; i < drhd->devices_cnt; i++)
3243                         if (drhd->devices[i] &&
3244                             !IS_GFX_DEVICE(drhd->devices[i]))
3245                                 break;
3246
3247                 if (i < drhd->devices_cnt)
3248                         continue;
3249
3250                 /* This IOMMU has *only* gfx devices. Either bypass it or
3251                    set the gfx_mapped flag, as appropriate */
3252                 if (dmar_map_gfx) {
3253                         intel_iommu_gfx_mapped = 1;
3254                 } else {
3255                         drhd->ignored = 1;
3256                         for (i = 0; i < drhd->devices_cnt; i++) {
3257                                 if (!drhd->devices[i])
3258                                         continue;
3259                                 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3260                         }
3261                 }
3262         }
3263 }
3264
3265 #ifdef CONFIG_SUSPEND
3266 static int init_iommu_hw(void)
3267 {
3268         struct dmar_drhd_unit *drhd;
3269         struct intel_iommu *iommu = NULL;
3270
3271         for_each_active_iommu(iommu, drhd)
3272                 if (iommu->qi)
3273                         dmar_reenable_qi(iommu);
3274
3275         for_each_iommu(iommu, drhd) {
3276                 if (drhd->ignored) {
3277                         /*
3278                          * we always have to disable PMRs or DMA may fail on
3279                          * this device
3280                          */
3281                         if (force_on)
3282                                 iommu_disable_protect_mem_regions(iommu);
3283                         continue;
3284                 }
3285
3286                 iommu_flush_write_buffer(iommu);
3287
3288                 iommu_set_root_entry(iommu);
3289
3290                 iommu->flush.flush_context(iommu, 0, 0, 0,
3291                                            DMA_CCMD_GLOBAL_INVL);
3292                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3293                                          DMA_TLB_GLOBAL_FLUSH);
3294                 if (iommu_enable_translation(iommu))
3295                         return 1;
3296                 iommu_disable_protect_mem_regions(iommu);
3297         }
3298
3299         return 0;
3300 }
3301
3302 static void iommu_flush_all(void)
3303 {
3304         struct dmar_drhd_unit *drhd;
3305         struct intel_iommu *iommu;
3306
3307         for_each_active_iommu(iommu, drhd) {
3308                 iommu->flush.flush_context(iommu, 0, 0, 0,
3309                                            DMA_CCMD_GLOBAL_INVL);
3310                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3311                                          DMA_TLB_GLOBAL_FLUSH);
3312         }
3313 }
3314
3315 static int iommu_suspend(void)
3316 {
3317         struct dmar_drhd_unit *drhd;
3318         struct intel_iommu *iommu = NULL;
3319         unsigned long flag;
3320
3321         for_each_active_iommu(iommu, drhd) {
3322                 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3323                                                  GFP_ATOMIC);
3324                 if (!iommu->iommu_state)
3325                         goto nomem;
3326         }
3327
3328         iommu_flush_all();
3329
3330         for_each_active_iommu(iommu, drhd) {
3331                 iommu_disable_translation(iommu);
3332
3333                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3334
3335                 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3336                         readl(iommu->reg + DMAR_FECTL_REG);
3337                 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3338                         readl(iommu->reg + DMAR_FEDATA_REG);
3339                 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3340                         readl(iommu->reg + DMAR_FEADDR_REG);
3341                 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3342                         readl(iommu->reg + DMAR_FEUADDR_REG);
3343
3344                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3345         }
3346         return 0;
3347
3348 nomem:
3349         for_each_active_iommu(iommu, drhd)
3350                 kfree(iommu->iommu_state);
3351
3352         return -ENOMEM;
3353 }
3354
3355 static void iommu_resume(void)
3356 {
3357         struct dmar_drhd_unit *drhd;
3358         struct intel_iommu *iommu = NULL;
3359         unsigned long flag;
3360
3361         if (init_iommu_hw()) {
3362                 if (force_on)
3363                         panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3364                 else
3365                         WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3366                 return;
3367         }
3368
3369         for_each_active_iommu(iommu, drhd) {
3370
3371                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3372
3373                 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3374                         iommu->reg + DMAR_FECTL_REG);
3375                 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3376                         iommu->reg + DMAR_FEDATA_REG);
3377                 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3378                         iommu->reg + DMAR_FEADDR_REG);
3379                 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3380                         iommu->reg + DMAR_FEUADDR_REG);
3381
3382                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3383         }
3384
3385         for_each_active_iommu(iommu, drhd)
3386                 kfree(iommu->iommu_state);
3387 }
3388
3389 static struct syscore_ops iommu_syscore_ops = {
3390         .resume         = iommu_resume,
3391         .suspend        = iommu_suspend,
3392 };
3393
3394 static void __init init_iommu_pm_ops(void)
3395 {
3396         register_syscore_ops(&iommu_syscore_ops);
3397 }
3398
3399 #else
3400 static inline void init_iommu_pm_ops(void) {}
3401 #endif  /* CONFIG_PM */
3402
3403 LIST_HEAD(dmar_rmrr_units);
3404
3405 static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
3406 {
3407         list_add(&rmrr->list, &dmar_rmrr_units);
3408 }
3409
3410
3411 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header)
3412 {
3413         struct acpi_dmar_reserved_memory *rmrr;
3414         struct dmar_rmrr_unit *rmrru;
3415
3416         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3417         if (!rmrru)
3418                 return -ENOMEM;
3419
3420         rmrru->hdr = header;
3421         rmrr = (struct acpi_dmar_reserved_memory *)header;
3422         rmrru->base_address = rmrr->base_address;
3423         rmrru->end_address = rmrr->end_address;
3424
3425         dmar_register_rmrr_unit(rmrru);
3426         return 0;
3427 }
3428
3429 static int __init
3430 rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
3431 {
3432         struct acpi_dmar_reserved_memory *rmrr;
3433         int ret;
3434
3435         rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
3436         ret = dmar_parse_dev_scope((void *)(rmrr + 1),
3437                 ((void *)rmrr) + rmrr->header.length,
3438                 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
3439
3440         if (ret || (rmrru->devices_cnt == 0)) {
3441                 list_del(&rmrru->list);
3442                 kfree(rmrru);
3443         }
3444         return ret;
3445 }
3446
3447 static LIST_HEAD(dmar_atsr_units);
3448
3449 int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
3450 {
3451         struct acpi_dmar_atsr *atsr;
3452         struct dmar_atsr_unit *atsru;
3453
3454         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3455         atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
3456         if (!atsru)
3457                 return -ENOMEM;
3458
3459         atsru->hdr = hdr;
3460         atsru->include_all = atsr->flags & 0x1;
3461
3462         list_add(&atsru->list, &dmar_atsr_units);
3463
3464         return 0;
3465 }
3466
3467 static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
3468 {
3469         int rc;
3470         struct acpi_dmar_atsr *atsr;
3471
3472         if (atsru->include_all)
3473                 return 0;
3474
3475         atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3476         rc = dmar_parse_dev_scope((void *)(atsr + 1),
3477                                 (void *)atsr + atsr->header.length,
3478                                 &atsru->devices_cnt, &atsru->devices,
3479                                 atsr->segment);
3480         if (rc || !atsru->devices_cnt) {
3481                 list_del(&atsru->list);
3482                 kfree(atsru);
3483         }
3484
3485         return rc;
3486 }
3487
3488 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
3489 {
3490         int i;
3491         struct pci_bus *bus;
3492         struct acpi_dmar_atsr *atsr;
3493         struct dmar_atsr_unit *atsru;
3494
3495         dev = pci_physfn(dev);
3496
3497         list_for_each_entry(atsru, &dmar_atsr_units, list) {
3498                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3499                 if (atsr->segment == pci_domain_nr(dev->bus))
3500                         goto found;
3501         }
3502
3503         return 0;
3504
3505 found:
3506         for (bus = dev->bus; bus; bus = bus->parent) {
3507                 struct pci_dev *bridge = bus->self;
3508
3509                 if (!bridge || !pci_is_pcie(bridge) ||
3510                     bridge->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
3511                         return 0;
3512
3513                 if (bridge->pcie_type == PCI_EXP_TYPE_ROOT_PORT) {
3514                         for (i = 0; i < atsru->devices_cnt; i++)
3515                                 if (atsru->devices[i] == bridge)
3516                                         return 1;
3517                         break;
3518                 }
3519         }
3520
3521         if (atsru->include_all)
3522                 return 1;
3523
3524         return 0;
3525 }
3526
3527 int dmar_parse_rmrr_atsr_dev(void)
3528 {
3529         struct dmar_rmrr_unit *rmrr, *rmrr_n;
3530         struct dmar_atsr_unit *atsr, *atsr_n;
3531         int ret = 0;
3532
3533         list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
3534                 ret = rmrr_parse_dev(rmrr);
3535                 if (ret)
3536                         return ret;
3537         }
3538
3539         list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
3540                 ret = atsr_parse_dev(atsr);
3541                 if (ret)
3542                         return ret;
3543         }
3544
3545         return ret;
3546 }
3547
3548 /*
3549  * Here we only respond to action of unbound device from driver.
3550  *
3551  * Added device is not attached to its DMAR domain here yet. That will happen
3552  * when mapping the device to iova.
3553  */
3554 static int device_notifier(struct notifier_block *nb,
3555                                   unsigned long action, void *data)
3556 {
3557         struct device *dev = data;
3558         struct pci_dev *pdev = to_pci_dev(dev);
3559         struct dmar_domain *domain;
3560
3561         if (iommu_no_mapping(dev))
3562                 return 0;
3563
3564         domain = find_domain(pdev);
3565         if (!domain)
3566                 return 0;
3567
3568         if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through) {
3569                 domain_remove_one_dev_info(domain, pdev);
3570
3571                 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3572                     !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
3573                     list_empty(&domain->devices))
3574                         domain_exit(domain);
3575         }
3576
3577         return 0;
3578 }
3579
3580 static struct notifier_block device_nb = {
3581         .notifier_call = device_notifier,
3582 };
3583
3584 int __init intel_iommu_init(void)
3585 {
3586         int ret = 0;
3587
3588         /* VT-d is required for a TXT/tboot launch, so enforce that */
3589         force_on = tboot_force_iommu();
3590
3591         if (dmar_table_init()) {
3592                 if (force_on)
3593                         panic("tboot: Failed to initialize DMAR table\n");
3594                 return  -ENODEV;
3595         }
3596
3597         if (dmar_dev_scope_init() < 0) {
3598                 if (force_on)
3599                         panic("tboot: Failed to initialize DMAR device scope\n");
3600                 return  -ENODEV;
3601         }
3602
3603         if (no_iommu || dmar_disabled)
3604                 return -ENODEV;
3605
3606         if (iommu_init_mempool()) {
3607                 if (force_on)
3608                         panic("tboot: Failed to initialize iommu memory\n");
3609                 return  -ENODEV;
3610         }
3611
3612         if (list_empty(&dmar_rmrr_units))
3613                 printk(KERN_INFO "DMAR: No RMRR found\n");
3614
3615         if (list_empty(&dmar_atsr_units))
3616                 printk(KERN_INFO "DMAR: No ATSR found\n");
3617
3618         if (dmar_init_reserved_ranges()) {
3619                 if (force_on)
3620                         panic("tboot: Failed to reserve iommu ranges\n");
3621                 return  -ENODEV;
3622         }
3623
3624         init_no_remapping_devices();
3625
3626         ret = init_dmars();
3627         if (ret) {
3628                 if (force_on)
3629                         panic("tboot: Failed to initialize DMARs\n");
3630                 printk(KERN_ERR "IOMMU: dmar init failed\n");
3631                 put_iova_domain(&reserved_iova_list);
3632                 iommu_exit_mempool();
3633                 return ret;
3634         }
3635         printk(KERN_INFO
3636         "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3637
3638         init_timer(&unmap_timer);
3639 #ifdef CONFIG_SWIOTLB
3640         swiotlb = 0;
3641 #endif
3642         dma_ops = &intel_dma_ops;
3643
3644         init_iommu_pm_ops();
3645
3646         bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
3647
3648         bus_register_notifier(&pci_bus_type, &device_nb);
3649
3650         return 0;
3651 }
3652
3653 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3654                                            struct pci_dev *pdev)
3655 {
3656         struct pci_dev *tmp, *parent;
3657
3658         if (!iommu || !pdev)
3659                 return;
3660
3661         /* dependent device detach */
3662         tmp = pci_find_upstream_pcie_bridge(pdev);
3663         /* Secondary interface's bus number and devfn 0 */
3664         if (tmp) {
3665                 parent = pdev->bus->self;
3666                 while (parent != tmp) {
3667                         iommu_detach_dev(iommu, parent->bus->number,
3668                                          parent->devfn);
3669                         parent = parent->bus->self;
3670                 }
3671                 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
3672                         iommu_detach_dev(iommu,
3673                                 tmp->subordinate->number, 0);
3674                 else /* this is a legacy PCI bridge */
3675                         iommu_detach_dev(iommu, tmp->bus->number,
3676                                          tmp->devfn);
3677         }
3678 }
3679
3680 static void domain_remove_one_dev_info(struct dmar_domain *domain,
3681                                           struct pci_dev *pdev)
3682 {
3683         struct device_domain_info *info;
3684         struct intel_iommu *iommu;
3685         unsigned long flags;
3686         int found = 0;
3687         struct list_head *entry, *tmp;
3688
3689         iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3690                                 pdev->devfn);
3691         if (!iommu)
3692                 return;
3693
3694         spin_lock_irqsave(&device_domain_lock, flags);
3695         list_for_each_safe(entry, tmp, &domain->devices) {
3696                 info = list_entry(entry, struct device_domain_info, link);
3697                 if (info->segment == pci_domain_nr(pdev->bus) &&
3698                     info->bus == pdev->bus->number &&
3699                     info->devfn == pdev->devfn) {
3700                         list_del(&info->link);
3701                         list_del(&info->global);
3702                         if (info->dev)
3703                                 info->dev->dev.archdata.iommu = NULL;
3704                         spin_unlock_irqrestore(&device_domain_lock, flags);
3705
3706                         iommu_disable_dev_iotlb(info);
3707                         iommu_detach_dev(iommu, info->bus, info->devfn);
3708                         iommu_detach_dependent_devices(iommu, pdev);
3709                         free_devinfo_mem(info);
3710
3711                         spin_lock_irqsave(&device_domain_lock, flags);
3712
3713                         if (found)
3714                                 break;
3715                         else
3716                                 continue;
3717                 }
3718
3719                 /* if there is no other devices under the same iommu
3720                  * owned by this domain, clear this iommu in iommu_bmp
3721                  * update iommu count and coherency
3722                  */
3723                 if (iommu == device_to_iommu(info->segment, info->bus,
3724                                             info->devfn))
3725                         found = 1;
3726         }
3727
3728         spin_unlock_irqrestore(&device_domain_lock, flags);
3729
3730         if (found == 0) {
3731                 unsigned long tmp_flags;
3732                 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
3733                 clear_bit(iommu->seq_id, &domain->iommu_bmp);
3734                 domain->iommu_count--;
3735                 domain_update_iommu_cap(domain);
3736                 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
3737
3738                 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3739                     !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)) {
3740                         spin_lock_irqsave(&iommu->lock, tmp_flags);
3741                         clear_bit(domain->id, iommu->domain_ids);
3742                         iommu->domains[domain->id] = NULL;
3743                         spin_unlock_irqrestore(&iommu->lock, tmp_flags);
3744                 }
3745         }
3746 }
3747
3748 static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3749 {
3750         struct device_domain_info *info;
3751         struct intel_iommu *iommu;
3752         unsigned long flags1, flags2;
3753
3754         spin_lock_irqsave(&device_domain_lock, flags1);
3755         while (!list_empty(&domain->devices)) {
3756                 info = list_entry(domain->devices.next,
3757                         struct device_domain_info, link);
3758                 list_del(&info->link);
3759                 list_del(&info->global);
3760                 if (info->dev)
3761                         info->dev->dev.archdata.iommu = NULL;
3762
3763                 spin_unlock_irqrestore(&device_domain_lock, flags1);
3764
3765                 iommu_disable_dev_iotlb(info);
3766                 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
3767                 iommu_detach_dev(iommu, info->bus, info->devfn);
3768                 iommu_detach_dependent_devices(iommu, info->dev);
3769
3770                 /* clear this iommu in iommu_bmp, update iommu count
3771                  * and capabilities
3772                  */
3773                 spin_lock_irqsave(&domain->iommu_lock, flags2);
3774                 if (test_and_clear_bit(iommu->seq_id,
3775                                        &domain->iommu_bmp)) {
3776                         domain->iommu_count--;
3777                         domain_update_iommu_cap(domain);
3778                 }
3779                 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3780
3781                 free_devinfo_mem(info);
3782                 spin_lock_irqsave(&device_domain_lock, flags1);
3783         }
3784         spin_unlock_irqrestore(&device_domain_lock, flags1);
3785 }
3786
3787 /* domain id for virtual machine, it won't be set in context */
3788 static unsigned long vm_domid;
3789
3790 static struct dmar_domain *iommu_alloc_vm_domain(void)
3791 {
3792         struct dmar_domain *domain;
3793
3794         domain = alloc_domain_mem();
3795         if (!domain)
3796                 return NULL;
3797
3798         domain->id = vm_domid++;
3799         domain->nid = -1;
3800         memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
3801         domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3802
3803         return domain;
3804 }
3805
3806 static int md_domain_init(struct dmar_domain *domain, int guest_width)
3807 {
3808         int adjust_width;
3809
3810         init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
3811         spin_lock_init(&domain->iommu_lock);
3812
3813         domain_reserve_special_ranges(domain);
3814
3815         /* calculate AGAW */
3816         domain->gaw = guest_width;
3817         adjust_width = guestwidth_to_adjustwidth(guest_width);
3818         domain->agaw = width_to_agaw(adjust_width);
3819
3820         INIT_LIST_HEAD(&domain->devices);
3821
3822         domain->iommu_count = 0;
3823         domain->iommu_coherency = 0;
3824         domain->iommu_snooping = 0;
3825         domain->iommu_superpage = 0;
3826         domain->max_addr = 0;
3827         domain->nid = -1;
3828
3829         /* always allocate the top pgd */
3830         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
3831         if (!domain->pgd)
3832                 return -ENOMEM;
3833         domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3834         return 0;
3835 }
3836
3837 static void iommu_free_vm_domain(struct dmar_domain *domain)
3838 {
3839         unsigned long flags;
3840         struct dmar_drhd_unit *drhd;
3841         struct intel_iommu *iommu;
3842         unsigned long i;
3843         unsigned long ndomains;
3844
3845         for_each_drhd_unit(drhd) {
3846                 if (drhd->ignored)
3847                         continue;
3848                 iommu = drhd->iommu;
3849
3850                 ndomains = cap_ndoms(iommu->cap);
3851                 for_each_set_bit(i, iommu->domain_ids, ndomains) {
3852                         if (iommu->domains[i] == domain) {
3853                                 spin_lock_irqsave(&iommu->lock, flags);
3854                                 clear_bit(i, iommu->domain_ids);
3855                                 iommu->domains[i] = NULL;
3856                                 spin_unlock_irqrestore(&iommu->lock, flags);
3857                                 break;
3858                         }
3859                 }
3860         }
3861 }
3862
3863 static void vm_domain_exit(struct dmar_domain *domain)
3864 {
3865         /* Domain 0 is reserved, so dont process it */
3866         if (!domain)
3867                 return;
3868
3869         vm_domain_remove_all_dev_info(domain);
3870         /* destroy iovas */
3871         put_iova_domain(&domain->iovad);
3872
3873         /* clear ptes */
3874         dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3875
3876         /* free page tables */
3877         dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3878
3879         iommu_free_vm_domain(domain);
3880         free_domain_mem(domain);
3881 }
3882
3883 static int intel_iommu_domain_init(struct iommu_domain *domain)
3884 {
3885         struct dmar_domain *dmar_domain;
3886
3887         dmar_domain = iommu_alloc_vm_domain();
3888         if (!dmar_domain) {
3889                 printk(KERN_ERR
3890                         "intel_iommu_domain_init: dmar_domain == NULL\n");
3891                 return -ENOMEM;
3892         }
3893         if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
3894                 printk(KERN_ERR
3895                         "intel_iommu_domain_init() failed\n");
3896                 vm_domain_exit(dmar_domain);
3897                 return -ENOMEM;
3898         }
3899         domain_update_iommu_cap(dmar_domain);
3900         domain->priv = dmar_domain;
3901
3902         return 0;
3903 }
3904
3905 static void intel_iommu_domain_destroy(struct iommu_domain *domain)
3906 {
3907         struct dmar_domain *dmar_domain = domain->priv;
3908
3909         domain->priv = NULL;
3910         vm_domain_exit(dmar_domain);
3911 }
3912
3913 static int intel_iommu_attach_device(struct iommu_domain *domain,
3914                                      struct device *dev)
3915 {
3916         struct dmar_domain *dmar_domain = domain->priv;
3917         struct pci_dev *pdev = to_pci_dev(dev);
3918         struct intel_iommu *iommu;
3919         int addr_width;
3920
3921         /* normally pdev is not mapped */
3922         if (unlikely(domain_context_mapped(pdev))) {
3923                 struct dmar_domain *old_domain;
3924
3925                 old_domain = find_domain(pdev);
3926                 if (old_domain) {
3927                         if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
3928                             dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
3929                                 domain_remove_one_dev_info(old_domain, pdev);
3930                         else
3931                                 domain_remove_dev_info(old_domain);
3932                 }
3933         }
3934
3935         iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3936                                 pdev->devfn);
3937         if (!iommu)
3938                 return -ENODEV;
3939
3940         /* check if this iommu agaw is sufficient for max mapped address */
3941         addr_width = agaw_to_width(iommu->agaw);
3942         if (addr_width > cap_mgaw(iommu->cap))
3943                 addr_width = cap_mgaw(iommu->cap);
3944
3945         if (dmar_domain->max_addr > (1LL << addr_width)) {
3946                 printk(KERN_ERR "%s: iommu width (%d) is not "
3947                        "sufficient for the mapped address (%llx)\n",
3948                        __func__, addr_width, dmar_domain->max_addr);
3949                 return -EFAULT;
3950         }
3951         dmar_domain->gaw = addr_width;
3952
3953         /*
3954          * Knock out extra levels of page tables if necessary
3955          */
3956         while (iommu->agaw < dmar_domain->agaw) {
3957                 struct dma_pte *pte;
3958
3959                 pte = dmar_domain->pgd;
3960                 if (dma_pte_present(pte)) {
3961                         dmar_domain->pgd = (struct dma_pte *)
3962                                 phys_to_virt(dma_pte_addr(pte));
3963                         free_pgtable_page(pte);
3964                 }
3965                 dmar_domain->agaw--;
3966         }
3967
3968         return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
3969 }
3970
3971 static void intel_iommu_detach_device(struct iommu_domain *domain,
3972                                       struct device *dev)
3973 {
3974         struct dmar_domain *dmar_domain = domain->priv;
3975         struct pci_dev *pdev = to_pci_dev(dev);
3976
3977         domain_remove_one_dev_info(dmar_domain, pdev);
3978 }
3979
3980 static int intel_iommu_map(struct iommu_domain *domain,
3981                            unsigned long iova, phys_addr_t hpa,
3982                            int gfp_order, int iommu_prot)
3983 {
3984         struct dmar_domain *dmar_domain = domain->priv;
3985         u64 max_addr;
3986         int prot = 0;
3987         size_t size;
3988         int ret;
3989
3990         if (iommu_prot & IOMMU_READ)
3991                 prot |= DMA_PTE_READ;
3992         if (iommu_prot & IOMMU_WRITE)
3993                 prot |= DMA_PTE_WRITE;
3994         if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
3995                 prot |= DMA_PTE_SNP;
3996
3997         size     = PAGE_SIZE << gfp_order;
3998         max_addr = iova + size;
3999         if (dmar_domain->max_addr < max_addr) {
4000                 u64 end;
4001
4002                 /* check if minimum agaw is sufficient for mapped address */
4003                 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
4004                 if (end < max_addr) {
4005                         printk(KERN_ERR "%s: iommu width (%d) is not "
4006                                "sufficient for the mapped address (%llx)\n",
4007                                __func__, dmar_domain->gaw, max_addr);
4008                         return -EFAULT;
4009                 }
4010                 dmar_domain->max_addr = max_addr;
4011         }
4012         /* Round up size to next multiple of PAGE_SIZE, if it and
4013            the low bits of hpa would take us onto the next page */
4014         size = aligned_nrpages(hpa, size);
4015         ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4016                                  hpa >> VTD_PAGE_SHIFT, size, prot);
4017         return ret;
4018 }
4019
4020 static int intel_iommu_unmap(struct iommu_domain *domain,
4021                              unsigned long iova, int gfp_order)
4022 {
4023         struct dmar_domain *dmar_domain = domain->priv;
4024         size_t size = PAGE_SIZE << gfp_order;
4025         int order;
4026
4027         order = dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
4028                             (iova + size - 1) >> VTD_PAGE_SHIFT);
4029
4030         if (dmar_domain->max_addr == iova + size)
4031                 dmar_domain->max_addr = iova;
4032
4033         return order;
4034 }
4035
4036 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
4037                                             unsigned long iova)
4038 {
4039         struct dmar_domain *dmar_domain = domain->priv;
4040         struct dma_pte *pte;
4041         u64 phys = 0;
4042
4043         pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, 0);
4044         if (pte)
4045                 phys = dma_pte_addr(pte);
4046
4047         return phys;
4048 }
4049
4050 static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
4051                                       unsigned long cap)
4052 {
4053         struct dmar_domain *dmar_domain = domain->priv;
4054
4055         if (cap == IOMMU_CAP_CACHE_COHERENCY)
4056                 return dmar_domain->iommu_snooping;
4057         if (cap == IOMMU_CAP_INTR_REMAP)
4058                 return intr_remapping_enabled;
4059
4060         return 0;
4061 }
4062
4063 static struct iommu_ops intel_iommu_ops = {
4064         .domain_init    = intel_iommu_domain_init,
4065         .domain_destroy = intel_iommu_domain_destroy,
4066         .attach_dev     = intel_iommu_attach_device,
4067         .detach_dev     = intel_iommu_detach_device,
4068         .map            = intel_iommu_map,
4069         .unmap          = intel_iommu_unmap,
4070         .iova_to_phys   = intel_iommu_iova_to_phys,
4071         .domain_has_cap = intel_iommu_domain_has_cap,
4072 };
4073
4074 static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
4075 {
4076         /*
4077          * Mobile 4 Series Chipset neglects to set RWBF capability,
4078          * but needs it:
4079          */
4080         printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
4081         rwbf_quirk = 1;
4082
4083         /* https://bugzilla.redhat.com/show_bug.cgi?id=538163 */
4084         if (dev->revision == 0x07) {
4085                 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
4086                 dmar_map_gfx = 0;
4087         }
4088 }
4089
4090 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
4091
4092 #define GGC 0x52
4093 #define GGC_MEMORY_SIZE_MASK    (0xf << 8)
4094 #define GGC_MEMORY_SIZE_NONE    (0x0 << 8)
4095 #define GGC_MEMORY_SIZE_1M      (0x1 << 8)
4096 #define GGC_MEMORY_SIZE_2M      (0x3 << 8)
4097 #define GGC_MEMORY_VT_ENABLED   (0x8 << 8)
4098 #define GGC_MEMORY_SIZE_2M_VT   (0x9 << 8)
4099 #define GGC_MEMORY_SIZE_3M_VT   (0xa << 8)
4100 #define GGC_MEMORY_SIZE_4M_VT   (0xb << 8)
4101
4102 static void __devinit quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
4103 {
4104         unsigned short ggc;
4105
4106         if (pci_read_config_word(dev, GGC, &ggc))
4107                 return;
4108
4109         if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
4110                 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4111                 dmar_map_gfx = 0;
4112         } else if (dmar_map_gfx) {
4113                 /* we have to ensure the gfx device is idle before we flush */
4114                 printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");
4115                 intel_iommu_strict = 1;
4116        }
4117 }
4118 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4119 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4120 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4121 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4122
4123 /* On Tylersburg chipsets, some BIOSes have been known to enable the
4124    ISOCH DMAR unit for the Azalia sound device, but not give it any
4125    TLB entries, which causes it to deadlock. Check for that.  We do
4126    this in a function called from init_dmars(), instead of in a PCI
4127    quirk, because we don't want to print the obnoxious "BIOS broken"
4128    message if VT-d is actually disabled.
4129 */
4130 static void __init check_tylersburg_isoch(void)
4131 {
4132         struct pci_dev *pdev;
4133         uint32_t vtisochctrl;
4134
4135         /* If there's no Azalia in the system anyway, forget it. */
4136         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4137         if (!pdev)
4138                 return;
4139         pci_dev_put(pdev);
4140
4141         /* System Management Registers. Might be hidden, in which case
4142            we can't do the sanity check. But that's OK, because the
4143            known-broken BIOSes _don't_ actually hide it, so far. */
4144         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4145         if (!pdev)
4146                 return;
4147
4148         if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4149                 pci_dev_put(pdev);
4150                 return;
4151         }
4152
4153         pci_dev_put(pdev);
4154
4155         /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4156         if (vtisochctrl & 1)
4157                 return;
4158
4159         /* Drop all bits other than the number of TLB entries */
4160         vtisochctrl &= 0x1c;
4161
4162         /* If we have the recommended number of TLB entries (16), fine. */
4163         if (vtisochctrl == 0x10)
4164                 return;
4165
4166         /* Zero TLB entries? You get to ride the short bus to school. */
4167         if (!vtisochctrl) {
4168                 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4169                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4170                      dmi_get_system_info(DMI_BIOS_VENDOR),
4171                      dmi_get_system_info(DMI_BIOS_VERSION),
4172                      dmi_get_system_info(DMI_PRODUCT_VERSION));
4173                 iommu_identity_mapping |= IDENTMAP_AZALIA;
4174                 return;
4175         }
4176
4177         printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4178                vtisochctrl);
4179 }