drivers/pci/intel-iommu.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  * Author: Fenghua Yu <fenghua.yu@intel.com>
  22  */
  23
  24 #include <linux/init.h>
  25 #include <linux/bitmap.h>
  26 #include <linux/debugfs.h>
  27 #include <linux/slab.h>
  28 #include <linux/irq.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/dma-mapping.h>
  34 #include <linux/mempool.h>
  35 #include <linux/timer.h>
  36 #include <linux/iova.h>
  37 #include <linux/iommu.h>
  38 #include <linux/intel-iommu.h>
  39 #include <linux/sysdev.h>
  40 #include <asm/cacheflush.h>
  41 #include <asm/iommu.h>
  42 #include "pci.h"
  43
  44 #define ROOT_SIZE               VTD_PAGE_SIZE
  45 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  46
  47 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  48 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  49
  50 #define IOAPIC_RANGE_START      (0xfee00000)
  51 #define IOAPIC_RANGE_END        (0xfeefffff)
  52 #define IOVA_START_ADDR         (0x1000)
  53
  54 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
  55
  56 #define MAX_AGAW_WIDTH 64
  57
  58 #define DOMAIN_MAX_ADDR(gaw) ((((u64)1) << gaw) - 1)
  59 #define DOMAIN_MAX_PFN(gaw)  ((((u64)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
  60
  61 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  62 #define DMA_32BIT_PFN           IOVA_PFN(DMA_BIT_MASK(32))
  63 #define DMA_64BIT_PFN           IOVA_PFN(DMA_BIT_MASK(64))
  64
  65 #ifndef PHYSICAL_PAGE_MASK
  66 #define PHYSICAL_PAGE_MASK PAGE_MASK
  67 #endif
  68
  69 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
  70    are never going to work. */
  71 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
  72 {
  73         return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
  74 }
  75
  76 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
  77 {
  78         return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
  79 }
  80 static inline unsigned long page_to_dma_pfn(struct page *pg)
  81 {
  82         return mm_to_dma_pfn(page_to_pfn(pg));
  83 }
  84 static inline unsigned long virt_to_dma_pfn(void *p)
  85 {
  86         return page_to_dma_pfn(virt_to_page(p));
  87 }
  88
  89 /* global iommu list, set NULL for ignored DMAR units */
  90 static struct intel_iommu **g_iommus;
  91
  92 static int rwbf_quirk;
  93
  94 /*
  95  * 0: Present
  96  * 1-11: Reserved
  97  * 12-63: Context Ptr (12 - (haw-1))
  98  * 64-127: Reserved
  99  */
 100 struct root_entry {
 101         u64     val;
 102         u64     rsvd1;
 103 };
 104 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
 105 static inline bool root_present(struct root_entry *root)
 106 {
 107         return (root->val & 1);
 108 }
 109 static inline void set_root_present(struct root_entry *root)
 110 {
 111         root->val |= 1;
 112 }
 113 static inline void set_root_value(struct root_entry *root, unsigned long value)
 114 {
 115         root->val |= value & VTD_PAGE_MASK;
 116 }
 117
 118 static inline struct context_entry *
 119 get_context_addr_from_root(struct root_entry *root)
 120 {
 121         return (struct context_entry *)
 122                 (root_present(root)?phys_to_virt(
 123                 root->val & VTD_PAGE_MASK) :
 124                 NULL);
 125 }
 126
 127 /*
 128  * low 64 bits:
 129  * 0: present
 130  * 1: fault processing disable
 131  * 2-3: translation type
 132  * 12-63: address space root
 133  * high 64 bits:
 134  * 0-2: address width
 135  * 3-6: aval
 136  * 8-23: domain id
 137  */
 138 struct context_entry {
 139         u64 lo;
 140         u64 hi;
 141 };
 142
 143 static inline bool context_present(struct context_entry *context)
 144 {
 145         return (context->lo & 1);
 146 }
 147 static inline void context_set_present(struct context_entry *context)
 148 {
 149         context->lo |= 1;
 150 }
 151
 152 static inline void context_set_fault_enable(struct context_entry *context)
 153 {
 154         context->lo &= (((u64)-1) << 2) | 1;
 155 }
 156
 157 static inline void context_set_translation_type(struct context_entry *context,
 158                                                 unsigned long value)
 159 {
 160         context->lo &= (((u64)-1) << 4) | 3;
 161         context->lo |= (value & 3) << 2;
 162 }
 163
 164 static inline void context_set_address_root(struct context_entry *context,
 165                                             unsigned long value)
 166 {
 167         context->lo |= value & VTD_PAGE_MASK;
 168 }
 169
 170 static inline void context_set_address_width(struct context_entry *context,
 171                                              unsigned long value)
 172 {
 173         context->hi |= value & 7;
 174 }
 175
 176 static inline void context_set_domain_id(struct context_entry *context,
 177                                          unsigned long value)
 178 {
 179         context->hi |= (value & ((1 << 16) - 1)) << 8;
 180 }
 181
 182 static inline void context_clear_entry(struct context_entry *context)
 183 {
 184         context->lo = 0;
 185         context->hi = 0;
 186 }
 187
 188 /*
 189  * 0: readable
 190  * 1: writable
 191  * 2-6: reserved
 192  * 7: super page
 193  * 8-10: available
 194  * 11: snoop behavior
 195  * 12-63: Host physcial address
 196  */
 197 struct dma_pte {
 198         u64 val;
 199 };
 200
 201 static inline void dma_clear_pte(struct dma_pte *pte)
 202 {
 203         pte->val = 0;
 204 }
 205
 206 static inline void dma_set_pte_readable(struct dma_pte *pte)
 207 {
 208         pte->val |= DMA_PTE_READ;
 209 }
 210
 211 static inline void dma_set_pte_writable(struct dma_pte *pte)
 212 {
 213         pte->val |= DMA_PTE_WRITE;
 214 }
 215
 216 static inline void dma_set_pte_snp(struct dma_pte *pte)
 217 {
 218         pte->val |= DMA_PTE_SNP;
 219 }
 220
 221 static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
 222 {
 223         pte->val = (pte->val & ~3) | (prot & 3);
 224 }
 225
 226 static inline u64 dma_pte_addr(struct dma_pte *pte)
 227 {
 228         return (pte->val & VTD_PAGE_MASK);
 229 }
 230
 231 static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
 232 {
 233         pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
 234 }
 235
 236 static inline bool dma_pte_present(struct dma_pte *pte)
 237 {
 238         return (pte->val & 3) != 0;
 239 }
 240
 241 /*
 242  * This domain is a statically identity mapping domain.
 243  *      1. This domain creats a static 1:1 mapping to all usable memory.
 244  *      2. It maps to each iommu if successful.
 245  *      3. Each iommu mapps to this domain if successful.
 246  */
 247 struct dmar_domain *si_domain;
 248
 249 /* devices under the same p2p bridge are owned in one domain */
 250 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
 251
 252 /* domain represents a virtual machine, more than one devices
 253  * across iommus may be owned in one domain, e.g. kvm guest.
 254  */
 255 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 1)
 256
 257 /* si_domain contains mulitple devices */
 258 #define DOMAIN_FLAG_STATIC_IDENTITY     (1 << 2)
 259
 260 struct dmar_domain {
 261         int     id;                     /* domain id */
 262         unsigned long iommu_bmp;        /* bitmap of iommus this domain uses*/
 263
 264         struct list_head devices;       /* all devices' list */
 265         struct iova_domain iovad;       /* iova's that belong to this domain */
 266
 267         struct dma_pte  *pgd;           /* virtual address */
 268         spinlock_t      mapping_lock;   /* page table lock */
 269         int             gaw;            /* max guest address width */
 270
 271         /* adjusted guest address width, 0 is level 2 30-bit */
 272         int             agaw;
 273
 274         int             flags;          /* flags to find out type of domain */
 275
 276         int             iommu_coherency;/* indicate coherency of iommu access */
 277         int             iommu_snooping; /* indicate snooping control feature*/
 278         int             iommu_count;    /* reference count of iommu */
 279         spinlock_t      iommu_lock;     /* protect iommu set in domain */
 280         u64             max_addr;       /* maximum mapped address */
 281 };
 282
 283 /* PCI domain-device relationship */
 284 struct device_domain_info {
 285         struct list_head link;  /* link to domain siblings */
 286         struct list_head global; /* link to global list */
 287         int segment;            /* PCI domain */
 288         u8 bus;                 /* PCI bus number */
 289         u8 devfn;               /* PCI devfn number */
 290         struct pci_dev *dev; /* it's NULL for PCIE-to-PCI bridge */
 291         struct intel_iommu *iommu; /* IOMMU used by this device */
 292         struct dmar_domain *domain; /* pointer to domain */
 293 };
 294
 295 static void flush_unmaps_timeout(unsigned long data);
 296
 297 DEFINE_TIMER(unmap_timer,  flush_unmaps_timeout, 0, 0);
 298
 299 #define HIGH_WATER_MARK 250
 300 struct deferred_flush_tables {
 301         int next;
 302         struct iova *iova[HIGH_WATER_MARK];
 303         struct dmar_domain *domain[HIGH_WATER_MARK];
 304 };
 305
 306 static struct deferred_flush_tables *deferred_flush;
 307
 308 /* bitmap for indexing intel_iommus */
 309 static int g_num_of_iommus;
 310
 311 static DEFINE_SPINLOCK(async_umap_flush_lock);
 312 static LIST_HEAD(unmaps_to_do);
 313
 314 static int timer_on;
 315 static long list_size;
 316
 317 static void domain_remove_dev_info(struct dmar_domain *domain);
 318
 319 #ifdef CONFIG_DMAR_DEFAULT_ON
 320 int dmar_disabled = 0;
 321 #else
 322 int dmar_disabled = 1;
 323 #endif /*CONFIG_DMAR_DEFAULT_ON*/
 324
 325 static int __initdata dmar_map_gfx = 1;
 326 static int dmar_forcedac;
 327 static int intel_iommu_strict;
 328
 329 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 330 static DEFINE_SPINLOCK(device_domain_lock);
 331 static LIST_HEAD(device_domain_list);
 332
 333 static struct iommu_ops intel_iommu_ops;
 334
 335 static int __init intel_iommu_setup(char *str)
 336 {
 337         if (!str)
 338                 return -EINVAL;
 339         while (*str) {
 340                 if (!strncmp(str, "on", 2)) {
 341                         dmar_disabled = 0;
 342                         printk(KERN_INFO "Intel-IOMMU: enabled\n");
 343                 } else if (!strncmp(str, "off", 3)) {
 344                         dmar_disabled = 1;
 345                         printk(KERN_INFO "Intel-IOMMU: disabled\n");
 346                 } else if (!strncmp(str, "igfx_off", 8)) {
 347                         dmar_map_gfx = 0;
 348                         printk(KERN_INFO
 349                                 "Intel-IOMMU: disable GFX device mapping\n");
 350                 } else if (!strncmp(str, "forcedac", 8)) {
 351                         printk(KERN_INFO
 352                                 "Intel-IOMMU: Forcing DAC for PCI devices\n");
 353                         dmar_forcedac = 1;
 354                 } else if (!strncmp(str, "strict", 6)) {
 355                         printk(KERN_INFO
 356                                 "Intel-IOMMU: disable batched IOTLB flush\n");
 357                         intel_iommu_strict = 1;
 358                 }
 359
 360                 str += strcspn(str, ",");
 361                 while (*str == ',')
 362                         str++;
 363         }
 364         return 0;
 365 }
 366 __setup("intel_iommu=", intel_iommu_setup);
 367
 368 static struct kmem_cache *iommu_domain_cache;
 369 static struct kmem_cache *iommu_devinfo_cache;
 370 static struct kmem_cache *iommu_iova_cache;
 371
 372 static inline void *iommu_kmem_cache_alloc(struct kmem_cache *cachep)
 373 {
 374         unsigned int flags;
 375         void *vaddr;
 376
 377         /* trying to avoid low memory issues */
 378         flags = current->flags & PF_MEMALLOC;
 379         current->flags |= PF_MEMALLOC;
 380         vaddr = kmem_cache_alloc(cachep, GFP_ATOMIC);
 381         current->flags &= (~PF_MEMALLOC | flags);
 382         return vaddr;
 383 }
 384
 385
 386 static inline void *alloc_pgtable_page(void)
 387 {
 388         unsigned int flags;
 389         void *vaddr;
 390
 391         /* trying to avoid low memory issues */
 392         flags = current->flags & PF_MEMALLOC;
 393         current->flags |= PF_MEMALLOC;
 394         vaddr = (void *)get_zeroed_page(GFP_ATOMIC);
 395         current->flags &= (~PF_MEMALLOC | flags);
 396         return vaddr;
 397 }
 398
 399 static inline void free_pgtable_page(void *vaddr)
 400 {
 401         free_page((unsigned long)vaddr);
 402 }
 403
 404 static inline void *alloc_domain_mem(void)
 405 {
 406         return iommu_kmem_cache_alloc(iommu_domain_cache);
 407 }
 408
 409 static void free_domain_mem(void *vaddr)
 410 {
 411         kmem_cache_free(iommu_domain_cache, vaddr);
 412 }
 413
 414 static inline void * alloc_devinfo_mem(void)
 415 {
 416         return iommu_kmem_cache_alloc(iommu_devinfo_cache);
 417 }
 418
 419 static inline void free_devinfo_mem(void *vaddr)
 420 {
 421         kmem_cache_free(iommu_devinfo_cache, vaddr);
 422 }
 423
 424 struct iova *alloc_iova_mem(void)
 425 {
 426         return iommu_kmem_cache_alloc(iommu_iova_cache);
 427 }
 428
 429 void free_iova_mem(struct iova *iova)
 430 {
 431         kmem_cache_free(iommu_iova_cache, iova);
 432 }
 433
 434
 435 static inline int width_to_agaw(int width);
 436
 437 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
 438 {
 439         unsigned long sagaw;
 440         int agaw = -1;
 441
 442         sagaw = cap_sagaw(iommu->cap);
 443         for (agaw = width_to_agaw(max_gaw);
 444              agaw >= 0; agaw--) {
 445                 if (test_bit(agaw, &sagaw))
 446                         break;
 447         }
 448
 449         return agaw;
 450 }
 451
 452 /*
 453  * Calculate max SAGAW for each iommu.
 454  */
 455 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
 456 {
 457         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
 458 }
 459
 460 /*
 461  * calculate agaw for each iommu.
 462  * "SAGAW" may be different across iommus, use a default agaw, and
 463  * get a supported less agaw for iommus that don't support the default agaw.
 464  */
 465 int iommu_calculate_agaw(struct intel_iommu *iommu)
 466 {
 467         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 468 }
 469
 470 /* This functionin only returns single iommu in a domain */
 471 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 472 {
 473         int iommu_id;
 474
 475         /* si_domain and vm domain should not get here. */
 476         BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
 477         BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
 478
 479         iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
 480         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 481                 return NULL;
 482
 483         return g_iommus[iommu_id];
 484 }
 485
 486 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 487 {
 488         int i;
 489
 490         domain->iommu_coherency = 1;
 491
 492         i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
 493         for (; i < g_num_of_iommus; ) {
 494                 if (!ecap_coherent(g_iommus[i]->ecap)) {
 495                         domain->iommu_coherency = 0;
 496                         break;
 497                 }
 498                 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
 499         }
 500 }
 501
 502 static void domain_update_iommu_snooping(struct dmar_domain *domain)
 503 {
 504         int i;
 505
 506         domain->iommu_snooping = 1;
 507
 508         i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
 509         for (; i < g_num_of_iommus; ) {
 510                 if (!ecap_sc_support(g_iommus[i]->ecap)) {
 511                         domain->iommu_snooping = 0;
 512                         break;
 513                 }
 514                 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
 515         }
 516 }
 517
 518 /* Some capabilities may be different across iommus */
 519 static void domain_update_iommu_cap(struct dmar_domain *domain)
 520 {
 521         domain_update_iommu_coherency(domain);
 522         domain_update_iommu_snooping(domain);
 523 }
 524
 525 static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
 526 {
 527         struct dmar_drhd_unit *drhd = NULL;
 528         int i;
 529
 530         for_each_drhd_unit(drhd) {
 531                 if (drhd->ignored)
 532                         continue;
 533                 if (segment != drhd->segment)
 534                         continue;
 535
 536                 for (i = 0; i < drhd->devices_cnt; i++) {
 537                         if (drhd->devices[i] &&
 538                             drhd->devices[i]->bus->number == bus &&
 539                             drhd->devices[i]->devfn == devfn)
 540                                 return drhd->iommu;
 541                         if (drhd->devices[i] &&
 542                             drhd->devices[i]->subordinate &&
 543                             drhd->devices[i]->subordinate->number <= bus &&
 544                             drhd->devices[i]->subordinate->subordinate >= bus)
 545                                 return drhd->iommu;
 546                 }
 547
 548                 if (drhd->include_all)
 549                         return drhd->iommu;
 550         }
 551
 552         return NULL;
 553 }
 554
 555 static void domain_flush_cache(struct dmar_domain *domain,
 556                                void *addr, int size)
 557 {
 558         if (!domain->iommu_coherency)
 559                 clflush_cache_range(addr, size);
 560 }
 561
 562 /* Gets context entry for a given bus and devfn */
 563 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
 564                 u8 bus, u8 devfn)
 565 {
 566         struct root_entry *root;
 567         struct context_entry *context;
 568         unsigned long phy_addr;
 569         unsigned long flags;
 570
 571         spin_lock_irqsave(&iommu->lock, flags);
 572         root = &iommu->root_entry[bus];
 573         context = get_context_addr_from_root(root);
 574         if (!context) {
 575                 context = (struct context_entry *)alloc_pgtable_page();
 576                 if (!context) {
 577                         spin_unlock_irqrestore(&iommu->lock, flags);
 578                         return NULL;
 579                 }
 580                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 581                 phy_addr = virt_to_phys((void *)context);
 582                 set_root_value(root, phy_addr);
 583                 set_root_present(root);
 584                 __iommu_flush_cache(iommu, root, sizeof(*root));
 585         }
 586         spin_unlock_irqrestore(&iommu->lock, flags);
 587         return &context[devfn];
 588 }
 589
 590 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 591 {
 592         struct root_entry *root;
 593         struct context_entry *context;
 594         int ret;
 595         unsigned long flags;
 596
 597         spin_lock_irqsave(&iommu->lock, flags);
 598         root = &iommu->root_entry[bus];
 599         context = get_context_addr_from_root(root);
 600         if (!context) {
 601                 ret = 0;
 602                 goto out;
 603         }
 604         ret = context_present(&context[devfn]);
 605 out:
 606         spin_unlock_irqrestore(&iommu->lock, flags);
 607         return ret;
 608 }
 609
 610 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
 611 {
 612         struct root_entry *root;
 613         struct context_entry *context;
 614         unsigned long flags;
 615
 616         spin_lock_irqsave(&iommu->lock, flags);
 617         root = &iommu->root_entry[bus];
 618         context = get_context_addr_from_root(root);
 619         if (context) {
 620                 context_clear_entry(&context[devfn]);
 621                 __iommu_flush_cache(iommu, &context[devfn], \
 622                         sizeof(*context));
 623         }
 624         spin_unlock_irqrestore(&iommu->lock, flags);
 625 }
 626
 627 static void free_context_table(struct intel_iommu *iommu)
 628 {
 629         struct root_entry *root;
 630         int i;
 631         unsigned long flags;
 632         struct context_entry *context;
 633
 634         spin_lock_irqsave(&iommu->lock, flags);
 635         if (!iommu->root_entry) {
 636                 goto out;
 637         }
 638         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 639                 root = &iommu->root_entry[i];
 640                 context = get_context_addr_from_root(root);
 641                 if (context)
 642                         free_pgtable_page(context);
 643         }
 644         free_pgtable_page(iommu->root_entry);
 645         iommu->root_entry = NULL;
 646 out:
 647         spin_unlock_irqrestore(&iommu->lock, flags);
 648 }
 649
 650 /* page table handling */
 651 #define LEVEL_STRIDE            (9)
 652 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
 653
 654 static inline int agaw_to_level(int agaw)
 655 {
 656         return agaw + 2;
 657 }
 658
 659 static inline int agaw_to_width(int agaw)
 660 {
 661         return 30 + agaw * LEVEL_STRIDE;
 662
 663 }
 664
 665 static inline int width_to_agaw(int width)
 666 {
 667         return (width - 30) / LEVEL_STRIDE;
 668 }
 669
 670 static inline unsigned int level_to_offset_bits(int level)
 671 {
 672         return (level - 1) * LEVEL_STRIDE;
 673 }
 674
 675 static inline int pfn_level_offset(unsigned long pfn, int level)
 676 {
 677         return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
 678 }
 679
 680 static inline unsigned long level_mask(int level)
 681 {
 682         return -1UL << level_to_offset_bits(level);
 683 }
 684
 685 static inline unsigned long level_size(int level)
 686 {
 687         return 1UL << level_to_offset_bits(level);
 688 }
 689
 690 static inline unsigned long align_to_level(unsigned long pfn, int level)
 691 {
 692         return (pfn + level_size(level) - 1) & level_mask(level);
 693 }
 694
 695 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
 696                                       unsigned long pfn)
 697 {
 698         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 699         struct dma_pte *parent, *pte = NULL;
 700         int level = agaw_to_level(domain->agaw);
 701         int offset;
 702         unsigned long flags;
 703
 704         BUG_ON(!domain->pgd);
 705         BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width);
 706         parent = domain->pgd;
 707
 708         spin_lock_irqsave(&domain->mapping_lock, flags);
 709         while (level > 0) {
 710                 void *tmp_page;
 711
 712                 offset = pfn_level_offset(pfn, level);
 713                 pte = &parent[offset];
 714                 if (level == 1)
 715                         break;
 716
 717                 if (!dma_pte_present(pte)) {
 718                         tmp_page = alloc_pgtable_page();
 719
 720                         if (!tmp_page) {
 721                                 spin_unlock_irqrestore(&domain->mapping_lock,
 722                                         flags);
 723                                 return NULL;
 724                         }
 725                         domain_flush_cache(domain, tmp_page, PAGE_SIZE);
 726                         dma_set_pte_pfn(pte, virt_to_dma_pfn(tmp_page));
 727                         /*
 728                          * high level table always sets r/w, last level page
 729                          * table control read/write
 730                          */
 731                         dma_set_pte_readable(pte);
 732                         dma_set_pte_writable(pte);
 733                         domain_flush_cache(domain, pte, sizeof(*pte));
 734                 }
 735                 parent = phys_to_virt(dma_pte_addr(pte));
 736                 level--;
 737         }
 738
 739         spin_unlock_irqrestore(&domain->mapping_lock, flags);
 740         return pte;
 741 }
 742
 743 /* return address's pte at specific level */
 744 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
 745                                          unsigned long pfn,
 746                                          int level)
 747 {
 748         struct dma_pte *parent, *pte = NULL;
 749         int total = agaw_to_level(domain->agaw);
 750         int offset;
 751
 752         parent = domain->pgd;
 753         while (level <= total) {
 754                 offset = pfn_level_offset(pfn, total);
 755                 pte = &parent[offset];
 756                 if (level == total)
 757                         return pte;
 758
 759                 if (!dma_pte_present(pte))
 760                         break;
 761                 parent = phys_to_virt(dma_pte_addr(pte));
 762                 total--;
 763         }
 764         return NULL;
 765 }
 766
 767 /* clear one page's page table */
 768 static void dma_pte_clear_one(struct dmar_domain *domain, unsigned long pfn)
 769 {
 770         struct dma_pte *pte = NULL;
 771
 772         /* get last level pte */
 773         pte = dma_pfn_level_pte(domain, pfn, 1);
 774
 775         if (pte) {
 776                 dma_clear_pte(pte);
 777                 domain_flush_cache(domain, pte, sizeof(*pte));
 778         }
 779 }
 780
 781 /* clear last level pte, a tlb flush should be followed */
 782 static void dma_pte_clear_range(struct dmar_domain *domain,
 783                                 unsigned long start_pfn,
 784                                 unsigned long last_pfn)
 785 {
 786         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 787
 788         BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
 789         BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
 790
 791         /* we don't need lock here; nobody else touches the iova range */
 792         while (start_pfn <= last_pfn) {
 793                 dma_pte_clear_one(domain, start_pfn);
 794                 start_pfn++;
 795         }
 796 }
 797
 798 /* free page table pages. last level pte should already be cleared */
 799 static void dma_pte_free_pagetable(struct dmar_domain *domain,
 800                                    unsigned long start_pfn,
 801                                    unsigned long last_pfn)
 802 {
 803         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 804         struct dma_pte *pte;
 805         int total = agaw_to_level(domain->agaw);
 806         int level;
 807         unsigned long tmp;
 808
 809         BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
 810         BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
 811
 812         /* we don't need lock here, nobody else touches the iova range */
 813         level = 2;
 814         while (level <= total) {
 815                 tmp = align_to_level(start_pfn, level);
 816
 817                 /* Only clear this pte/pmd if we're asked to clear its
 818                    _whole_ range */
 819                 if (tmp + level_size(level) - 1 > last_pfn)
 820                         return;
 821
 822                 while (tmp <= last_pfn) {
 823                         pte = dma_pfn_level_pte(domain, tmp, level);
 824                         if (pte) {
 825                                 free_pgtable_page(
 826                                         phys_to_virt(dma_pte_addr(pte)));
 827                                 dma_clear_pte(pte);
 828                                 domain_flush_cache(domain, pte, sizeof(*pte));
 829                         }
 830                         tmp += level_size(level);
 831                 }
 832                 level++;
 833         }
 834         /* free pgd */
 835         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
 836                 free_pgtable_page(domain->pgd);
 837                 domain->pgd = NULL;
 838         }
 839 }
 840
 841 /* iommu handling */
 842 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
 843 {
 844         struct root_entry *root;
 845         unsigned long flags;
 846
 847         root = (struct root_entry *)alloc_pgtable_page();
 848         if (!root)
 849                 return -ENOMEM;
 850
 851         __iommu_flush_cache(iommu, root, ROOT_SIZE);
 852
 853         spin_lock_irqsave(&iommu->lock, flags);
 854         iommu->root_entry = root;
 855         spin_unlock_irqrestore(&iommu->lock, flags);
 856
 857         return 0;
 858 }
 859
 860 static void iommu_set_root_entry(struct intel_iommu *iommu)
 861 {
 862         void *addr;
 863         u32 sts;
 864         unsigned long flag;
 865
 866         addr = iommu->root_entry;
 867
 868         spin_lock_irqsave(&iommu->register_lock, flag);
 869         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
 870
 871         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
 872
 873         /* Make sure hardware complete it */
 874         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 875                       readl, (sts & DMA_GSTS_RTPS), sts);
 876
 877         spin_unlock_irqrestore(&iommu->register_lock, flag);
 878 }
 879
 880 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
 881 {
 882         u32 val;
 883         unsigned long flag;
 884
 885         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
 886                 return;
 887
 888         spin_lock_irqsave(&iommu->register_lock, flag);
 889         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
 890
 891         /* Make sure hardware complete it */
 892         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 893                       readl, (!(val & DMA_GSTS_WBFS)), val);
 894
 895         spin_unlock_irqrestore(&iommu->register_lock, flag);
 896 }
 897
 898 /* return value determine if we need a write buffer flush */
 899 static void __iommu_flush_context(struct intel_iommu *iommu,
 900                                   u16 did, u16 source_id, u8 function_mask,
 901                                   u64 type)
 902 {
 903         u64 val = 0;
 904         unsigned long flag;
 905
 906         switch (type) {
 907         case DMA_CCMD_GLOBAL_INVL:
 908                 val = DMA_CCMD_GLOBAL_INVL;
 909                 break;
 910         case DMA_CCMD_DOMAIN_INVL:
 911                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
 912                 break;
 913         case DMA_CCMD_DEVICE_INVL:
 914                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
 915                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
 916                 break;
 917         default:
 918                 BUG();
 919         }
 920         val |= DMA_CCMD_ICC;
 921
 922         spin_lock_irqsave(&iommu->register_lock, flag);
 923         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
 924
 925         /* Make sure hardware complete it */
 926         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
 927                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
 928
 929         spin_unlock_irqrestore(&iommu->register_lock, flag);
 930 }
 931
 932 /* return value determine if we need a write buffer flush */
 933 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
 934                                 u64 addr, unsigned int size_order, u64 type)
 935 {
 936         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
 937         u64 val = 0, val_iva = 0;
 938         unsigned long flag;
 939
 940         switch (type) {
 941         case DMA_TLB_GLOBAL_FLUSH:
 942                 /* global flush doesn't need set IVA_REG */
 943                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
 944                 break;
 945         case DMA_TLB_DSI_FLUSH:
 946                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
 947                 break;
 948         case DMA_TLB_PSI_FLUSH:
 949                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
 950                 /* Note: always flush non-leaf currently */
 951                 val_iva = size_order | addr;
 952                 break;
 953         default:
 954                 BUG();
 955         }
 956         /* Note: set drain read/write */
 957 #if 0
 958         /*
 959          * This is probably to be super secure.. Looks like we can
 960          * ignore it without any impact.
 961          */
 962         if (cap_read_drain(iommu->cap))
 963                 val |= DMA_TLB_READ_DRAIN;
 964 #endif
 965         if (cap_write_drain(iommu->cap))
 966                 val |= DMA_TLB_WRITE_DRAIN;
 967
 968         spin_lock_irqsave(&iommu->register_lock, flag);
 969         /* Note: Only uses first TLB reg currently */
 970         if (val_iva)
 971                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
 972         dmar_writeq(iommu->reg + tlb_offset + 8, val);
 973
 974         /* Make sure hardware complete it */
 975         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
 976                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
 977
 978         spin_unlock_irqrestore(&iommu->register_lock, flag);
 979
 980         /* check IOTLB invalidation granularity */
 981         if (DMA_TLB_IAIG(val) == 0)
 982                 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
 983         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
 984                 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
 985                         (unsigned long long)DMA_TLB_IIRG(type),
 986                         (unsigned long long)DMA_TLB_IAIG(val));
 987 }
 988
 989 static struct device_domain_info *iommu_support_dev_iotlb(
 990         struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
 991 {
 992         int found = 0;
 993         unsigned long flags;
 994         struct device_domain_info *info;
 995         struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
 996
 997         if (!ecap_dev_iotlb_support(iommu->ecap))
 998                 return NULL;
 999
1000         if (!iommu->qi)
1001                 return NULL;
1002
1003         spin_lock_irqsave(&device_domain_lock, flags);
1004         list_for_each_entry(info, &domain->devices, link)
1005                 if (info->bus == bus && info->devfn == devfn) {
1006                         found = 1;
1007                         break;
1008                 }
1009         spin_unlock_irqrestore(&device_domain_lock, flags);
1010
1011         if (!found || !info->dev)
1012                 return NULL;
1013
1014         if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1015                 return NULL;
1016
1017         if (!dmar_find_matched_atsr_unit(info->dev))
1018                 return NULL;
1019
1020         info->iommu = iommu;
1021
1022         return info;
1023 }
1024
1025 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1026 {
1027         if (!info)
1028                 return;
1029
1030         pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1031 }
1032
1033 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1034 {
1035         if (!info->dev || !pci_ats_enabled(info->dev))
1036                 return;
1037
1038         pci_disable_ats(info->dev);
1039 }
1040
1041 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1042                                   u64 addr, unsigned mask)
1043 {
1044         u16 sid, qdep;
1045         unsigned long flags;
1046         struct device_domain_info *info;
1047
1048         spin_lock_irqsave(&device_domain_lock, flags);
1049         list_for_each_entry(info, &domain->devices, link) {
1050                 if (!info->dev || !pci_ats_enabled(info->dev))
1051                         continue;
1052
1053                 sid = info->bus << 8 | info->devfn;
1054                 qdep = pci_ats_queue_depth(info->dev);
1055                 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1056         }
1057         spin_unlock_irqrestore(&device_domain_lock, flags);
1058 }
1059
1060 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1061                                   unsigned long pfn, unsigned int pages)
1062 {
1063         unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1064         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1065
1066         BUG_ON(pages == 0);
1067
1068         /*
1069          * Fallback to domain selective flush if no PSI support or the size is
1070          * too big.
1071          * PSI requires page size to be 2 ^ x, and the base address is naturally
1072          * aligned to the size
1073          */
1074         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1075                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1076                                                 DMA_TLB_DSI_FLUSH);
1077         else
1078                 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1079                                                 DMA_TLB_PSI_FLUSH);
1080
1081         /*
1082          * In caching mode, domain ID 0 is reserved for non-present to present
1083          * mapping flush. Device IOTLB doesn't need to be flushed in this case.
1084          */
1085         if (!cap_caching_mode(iommu->cap) || did)
1086                 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
1087 }
1088
1089 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1090 {
1091         u32 pmen;
1092         unsigned long flags;
1093
1094         spin_lock_irqsave(&iommu->register_lock, flags);
1095         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1096         pmen &= ~DMA_PMEN_EPM;
1097         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1098
1099         /* wait for the protected region status bit to clear */
1100         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1101                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1102
1103         spin_unlock_irqrestore(&iommu->register_lock, flags);
1104 }
1105
1106 static int iommu_enable_translation(struct intel_iommu *iommu)
1107 {
1108         u32 sts;
1109         unsigned long flags;
1110
1111         spin_lock_irqsave(&iommu->register_lock, flags);
1112         iommu->gcmd |= DMA_GCMD_TE;
1113         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1114
1115         /* Make sure hardware complete it */
1116         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1117                       readl, (sts & DMA_GSTS_TES), sts);
1118
1119         spin_unlock_irqrestore(&iommu->register_lock, flags);
1120         return 0;
1121 }
1122
1123 static int iommu_disable_translation(struct intel_iommu *iommu)
1124 {
1125         u32 sts;
1126         unsigned long flag;
1127
1128         spin_lock_irqsave(&iommu->register_lock, flag);
1129         iommu->gcmd &= ~DMA_GCMD_TE;
1130         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1131
1132         /* Make sure hardware complete it */
1133         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1134                       readl, (!(sts & DMA_GSTS_TES)), sts);
1135
1136         spin_unlock_irqrestore(&iommu->register_lock, flag);
1137         return 0;
1138 }
1139
1140
1141 static int iommu_init_domains(struct intel_iommu *iommu)
1142 {
1143         unsigned long ndomains;
1144         unsigned long nlongs;
1145
1146         ndomains = cap_ndoms(iommu->cap);
1147         pr_debug("Number of Domains supportd <%ld>\n", ndomains);
1148         nlongs = BITS_TO_LONGS(ndomains);
1149
1150         /* TBD: there might be 64K domains,
1151          * consider other allocation for future chip
1152          */
1153         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1154         if (!iommu->domain_ids) {
1155                 printk(KERN_ERR "Allocating domain id array failed\n");
1156                 return -ENOMEM;
1157         }
1158         iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1159                         GFP_KERNEL);
1160         if (!iommu->domains) {
1161                 printk(KERN_ERR "Allocating domain array failed\n");
1162                 kfree(iommu->domain_ids);
1163                 return -ENOMEM;
1164         }
1165
1166         spin_lock_init(&iommu->lock);
1167
1168         /*
1169          * if Caching mode is set, then invalid translations are tagged
1170          * with domainid 0. Hence we need to pre-allocate it.
1171          */
1172         if (cap_caching_mode(iommu->cap))
1173                 set_bit(0, iommu->domain_ids);
1174         return 0;
1175 }
1176
1177
1178 static void domain_exit(struct dmar_domain *domain);
1179 static void vm_domain_exit(struct dmar_domain *domain);
1180
1181 void free_dmar_iommu(struct intel_iommu *iommu)
1182 {
1183         struct dmar_domain *domain;
1184         int i;
1185         unsigned long flags;
1186
1187         i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
1188         for (; i < cap_ndoms(iommu->cap); ) {
1189                 domain = iommu->domains[i];
1190                 clear_bit(i, iommu->domain_ids);
1191
1192                 spin_lock_irqsave(&domain->iommu_lock, flags);
1193                 if (--domain->iommu_count == 0) {
1194                         if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1195                                 vm_domain_exit(domain);
1196                         else
1197                                 domain_exit(domain);
1198                 }
1199                 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1200
1201                 i = find_next_bit(iommu->domain_ids,
1202                         cap_ndoms(iommu->cap), i+1);
1203         }
1204
1205         if (iommu->gcmd & DMA_GCMD_TE)
1206                 iommu_disable_translation(iommu);
1207
1208         if (iommu->irq) {
1209                 set_irq_data(iommu->irq, NULL);
1210                 /* This will mask the irq */
1211                 free_irq(iommu->irq, iommu);
1212                 destroy_irq(iommu->irq);
1213         }
1214
1215         kfree(iommu->domains);
1216         kfree(iommu->domain_ids);
1217
1218         g_iommus[iommu->seq_id] = NULL;
1219
1220         /* if all iommus are freed, free g_iommus */
1221         for (i = 0; i < g_num_of_iommus; i++) {
1222                 if (g_iommus[i])
1223                         break;
1224         }
1225
1226         if (i == g_num_of_iommus)
1227                 kfree(g_iommus);
1228
1229         /* free context mapping */
1230         free_context_table(iommu);
1231 }
1232
1233 static struct dmar_domain *alloc_domain(void)
1234 {
1235         struct dmar_domain *domain;
1236
1237         domain = alloc_domain_mem();
1238         if (!domain)
1239                 return NULL;
1240
1241         memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
1242         domain->flags = 0;
1243
1244         return domain;
1245 }
1246
1247 static int iommu_attach_domain(struct dmar_domain *domain,
1248                                struct intel_iommu *iommu)
1249 {
1250         int num;
1251         unsigned long ndomains;
1252         unsigned long flags;
1253
1254         ndomains = cap_ndoms(iommu->cap);
1255
1256         spin_lock_irqsave(&iommu->lock, flags);
1257
1258         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1259         if (num >= ndomains) {
1260                 spin_unlock_irqrestore(&iommu->lock, flags);
1261                 printk(KERN_ERR "IOMMU: no free domain ids\n");
1262                 return -ENOMEM;
1263         }
1264
1265         domain->id = num;
1266         set_bit(num, iommu->domain_ids);
1267         set_bit(iommu->seq_id, &domain->iommu_bmp);
1268         iommu->domains[num] = domain;
1269         spin_unlock_irqrestore(&iommu->lock, flags);
1270
1271         return 0;
1272 }
1273
1274 static void iommu_detach_domain(struct dmar_domain *domain,
1275                                 struct intel_iommu *iommu)
1276 {
1277         unsigned long flags;
1278         int num, ndomains;
1279         int found = 0;
1280
1281         spin_lock_irqsave(&iommu->lock, flags);
1282         ndomains = cap_ndoms(iommu->cap);
1283         num = find_first_bit(iommu->domain_ids, ndomains);
1284         for (; num < ndomains; ) {
1285                 if (iommu->domains[num] == domain) {
1286                         found = 1;
1287                         break;
1288                 }
1289                 num = find_next_bit(iommu->domain_ids,
1290                                     cap_ndoms(iommu->cap), num+1);
1291         }
1292
1293         if (found) {
1294                 clear_bit(num, iommu->domain_ids);
1295                 clear_bit(iommu->seq_id, &domain->iommu_bmp);
1296                 iommu->domains[num] = NULL;
1297         }
1298         spin_unlock_irqrestore(&iommu->lock, flags);
1299 }
1300
1301 static struct iova_domain reserved_iova_list;
1302 static struct lock_class_key reserved_alloc_key;
1303 static struct lock_class_key reserved_rbtree_key;
1304
1305 static void dmar_init_reserved_ranges(void)
1306 {
1307         struct pci_dev *pdev = NULL;
1308         struct iova *iova;
1309         int i;
1310         u64 addr, size;
1311
1312         init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1313
1314         lockdep_set_class(&reserved_iova_list.iova_alloc_lock,
1315                 &reserved_alloc_key);
1316         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1317                 &reserved_rbtree_key);
1318
1319         /* IOAPIC ranges shouldn't be accessed by DMA */
1320         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1321                 IOVA_PFN(IOAPIC_RANGE_END));
1322         if (!iova)
1323                 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1324
1325         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1326         for_each_pci_dev(pdev) {
1327                 struct resource *r;
1328
1329                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1330                         r = &pdev->resource[i];
1331                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1332                                 continue;
1333                         addr = r->start;
1334                         addr &= PHYSICAL_PAGE_MASK;
1335                         size = r->end - addr;
1336                         size = PAGE_ALIGN(size);
1337                         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(addr),
1338                                 IOVA_PFN(size + addr) - 1);
1339                         if (!iova)
1340                                 printk(KERN_ERR "Reserve iova failed\n");
1341                 }
1342         }
1343
1344 }
1345
1346 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1347 {
1348         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1349 }
1350
1351 static inline int guestwidth_to_adjustwidth(int gaw)
1352 {
1353         int agaw;
1354         int r = (gaw - 12) % 9;
1355
1356         if (r == 0)
1357                 agaw = gaw;
1358         else
1359                 agaw = gaw + 9 - r;
1360         if (agaw > 64)
1361                 agaw = 64;
1362         return agaw;
1363 }
1364
1365 static int domain_init(struct dmar_domain *domain, int guest_width)
1366 {
1367         struct intel_iommu *iommu;
1368         int adjust_width, agaw;
1369         unsigned long sagaw;
1370
1371         init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1372         spin_lock_init(&domain->mapping_lock);
1373         spin_lock_init(&domain->iommu_lock);
1374
1375         domain_reserve_special_ranges(domain);
1376
1377         /* calculate AGAW */
1378         iommu = domain_get_iommu(domain);
1379         if (guest_width > cap_mgaw(iommu->cap))
1380                 guest_width = cap_mgaw(iommu->cap);
1381         domain->gaw = guest_width;
1382         adjust_width = guestwidth_to_adjustwidth(guest_width);
1383         agaw = width_to_agaw(adjust_width);
1384         sagaw = cap_sagaw(iommu->cap);
1385         if (!test_bit(agaw, &sagaw)) {
1386                 /* hardware doesn't support it, choose a bigger one */
1387                 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1388                 agaw = find_next_bit(&sagaw, 5, agaw);
1389                 if (agaw >= 5)
1390                         return -ENODEV;
1391         }
1392         domain->agaw = agaw;
1393         INIT_LIST_HEAD(&domain->devices);
1394
1395         if (ecap_coherent(iommu->ecap))
1396                 domain->iommu_coherency = 1;
1397         else
1398                 domain->iommu_coherency = 0;
1399
1400         if (ecap_sc_support(iommu->ecap))
1401                 domain->iommu_snooping = 1;
1402         else
1403                 domain->iommu_snooping = 0;
1404
1405         domain->iommu_count = 1;
1406
1407         /* always allocate the top pgd */
1408         domain->pgd = (struct dma_pte *)alloc_pgtable_page();
1409         if (!domain->pgd)
1410                 return -ENOMEM;
1411         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1412         return 0;
1413 }
1414
1415 static void domain_exit(struct dmar_domain *domain)
1416 {
1417         struct dmar_drhd_unit *drhd;
1418         struct intel_iommu *iommu;
1419
1420         /* Domain 0 is reserved, so dont process it */
1421         if (!domain)
1422                 return;
1423
1424         domain_remove_dev_info(domain);
1425         /* destroy iovas */
1426         put_iova_domain(&domain->iovad);
1427
1428         /* clear ptes */
1429         dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1430
1431         /* free page tables */
1432         dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1433
1434         for_each_active_iommu(iommu, drhd)
1435                 if (test_bit(iommu->seq_id, &domain->iommu_bmp))
1436                         iommu_detach_domain(domain, iommu);
1437
1438         free_domain_mem(domain);
1439 }
1440
1441 static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1442                                  u8 bus, u8 devfn, int translation)
1443 {
1444         struct context_entry *context;
1445         unsigned long flags;
1446         struct intel_iommu *iommu;
1447         struct dma_pte *pgd;
1448         unsigned long num;
1449         unsigned long ndomains;
1450         int id;
1451         int agaw;
1452         struct device_domain_info *info = NULL;
1453
1454         pr_debug("Set context mapping for %02x:%02x.%d\n",
1455                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1456
1457         BUG_ON(!domain->pgd);
1458         BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1459                translation != CONTEXT_TT_MULTI_LEVEL);
1460
1461         iommu = device_to_iommu(segment, bus, devfn);
1462         if (!iommu)
1463                 return -ENODEV;
1464
1465         context = device_to_context_entry(iommu, bus, devfn);
1466         if (!context)
1467                 return -ENOMEM;
1468         spin_lock_irqsave(&iommu->lock, flags);
1469         if (context_present(context)) {
1470                 spin_unlock_irqrestore(&iommu->lock, flags);
1471                 return 0;
1472         }
1473
1474         id = domain->id;
1475         pgd = domain->pgd;
1476
1477         if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1478             domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
1479                 int found = 0;
1480
1481                 /* find an available domain id for this device in iommu */
1482                 ndomains = cap_ndoms(iommu->cap);
1483                 num = find_first_bit(iommu->domain_ids, ndomains);
1484                 for (; num < ndomains; ) {
1485                         if (iommu->domains[num] == domain) {
1486                                 id = num;
1487                                 found = 1;
1488                                 break;
1489                         }
1490                         num = find_next_bit(iommu->domain_ids,
1491                                             cap_ndoms(iommu->cap), num+1);
1492                 }
1493
1494                 if (found == 0) {
1495                         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1496                         if (num >= ndomains) {
1497                                 spin_unlock_irqrestore(&iommu->lock, flags);
1498                                 printk(KERN_ERR "IOMMU: no free domain ids\n");
1499                                 return -EFAULT;
1500                         }
1501
1502                         set_bit(num, iommu->domain_ids);
1503                         set_bit(iommu->seq_id, &domain->iommu_bmp);
1504                         iommu->domains[num] = domain;
1505                         id = num;
1506                 }
1507
1508                 /* Skip top levels of page tables for
1509                  * iommu which has less agaw than default.
1510                  */
1511                 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1512                         pgd = phys_to_virt(dma_pte_addr(pgd));
1513                         if (!dma_pte_present(pgd)) {
1514                                 spin_unlock_irqrestore(&iommu->lock, flags);
1515                                 return -ENOMEM;
1516                         }
1517                 }
1518         }
1519
1520         context_set_domain_id(context, id);
1521
1522         if (translation != CONTEXT_TT_PASS_THROUGH) {
1523                 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1524                 translation = info ? CONTEXT_TT_DEV_IOTLB :
1525                                      CONTEXT_TT_MULTI_LEVEL;
1526         }
1527         /*
1528          * In pass through mode, AW must be programmed to indicate the largest
1529          * AGAW value supported by hardware. And ASR is ignored by hardware.
1530          */
1531         if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
1532                 context_set_address_width(context, iommu->msagaw);
1533         else {
1534                 context_set_address_root(context, virt_to_phys(pgd));
1535                 context_set_address_width(context, iommu->agaw);
1536         }
1537
1538         context_set_translation_type(context, translation);
1539         context_set_fault_enable(context);
1540         context_set_present(context);
1541         domain_flush_cache(domain, context, sizeof(*context));
1542
1543         /*
1544          * It's a non-present to present mapping. If hardware doesn't cache
1545          * non-present entry we only need to flush the write-buffer. If the
1546          * _does_ cache non-present entries, then it does so in the special
1547          * domain #0, which we have to flush:
1548          */
1549         if (cap_caching_mode(iommu->cap)) {
1550                 iommu->flush.flush_context(iommu, 0,
1551                                            (((u16)bus) << 8) | devfn,
1552                                            DMA_CCMD_MASK_NOBIT,
1553                                            DMA_CCMD_DEVICE_INVL);
1554                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_DSI_FLUSH);
1555         } else {
1556                 iommu_flush_write_buffer(iommu);
1557         }
1558         iommu_enable_dev_iotlb(info);
1559         spin_unlock_irqrestore(&iommu->lock, flags);
1560
1561         spin_lock_irqsave(&domain->iommu_lock, flags);
1562         if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {
1563                 domain->iommu_count++;
1564                 domain_update_iommu_cap(domain);
1565         }
1566         spin_unlock_irqrestore(&domain->iommu_lock, flags);
1567         return 0;
1568 }
1569
1570 static int
1571 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1572                         int translation)
1573 {
1574         int ret;
1575         struct pci_dev *tmp, *parent;
1576
1577         ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
1578                                          pdev->bus->number, pdev->devfn,
1579                                          translation);
1580         if (ret)
1581                 return ret;
1582
1583         /* dependent device mapping */
1584         tmp = pci_find_upstream_pcie_bridge(pdev);
1585         if (!tmp)
1586                 return 0;
1587         /* Secondary interface's bus number and devfn 0 */
1588         parent = pdev->bus->self;
1589         while (parent != tmp) {
1590                 ret = domain_context_mapping_one(domain,
1591                                                  pci_domain_nr(parent->bus),
1592                                                  parent->bus->number,
1593                                                  parent->devfn, translation);
1594                 if (ret)
1595                         return ret;
1596                 parent = parent->bus->self;
1597         }
1598         if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
1599                 return domain_context_mapping_one(domain,
1600                                         pci_domain_nr(tmp->subordinate),
1601                                         tmp->subordinate->number, 0,
1602                                         translation);
1603         else /* this is a legacy PCI bridge */
1604                 return domain_context_mapping_one(domain,
1605                                                   pci_domain_nr(tmp->bus),
1606                                                   tmp->bus->number,
1607                                                   tmp->devfn,
1608                                                   translation);
1609 }
1610
1611 static int domain_context_mapped(struct pci_dev *pdev)
1612 {
1613         int ret;
1614         struct pci_dev *tmp, *parent;
1615         struct intel_iommu *iommu;
1616
1617         iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1618                                 pdev->devfn);
1619         if (!iommu)
1620                 return -ENODEV;
1621
1622         ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
1623         if (!ret)
1624                 return ret;
1625         /* dependent device mapping */
1626         tmp = pci_find_upstream_pcie_bridge(pdev);
1627         if (!tmp)
1628                 return ret;
1629         /* Secondary interface's bus number and devfn 0 */
1630         parent = pdev->bus->self;
1631         while (parent != tmp) {
1632                 ret = device_context_mapped(iommu, parent->bus->number,
1633                                             parent->devfn);
1634                 if (!ret)
1635                         return ret;
1636                 parent = parent->bus->self;
1637         }
1638         if (tmp->is_pcie)
1639                 return device_context_mapped(iommu, tmp->subordinate->number,
1640                                              0);
1641         else
1642                 return device_context_mapped(iommu, tmp->bus->number,
1643                                              tmp->devfn);
1644 }
1645
1646 static int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1647                               unsigned long phys_pfn, unsigned long nr_pages,
1648                               int prot)
1649 {
1650         struct dma_pte *pte;
1651         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1652
1653         BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1654
1655         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1656                 return -EINVAL;
1657
1658         while (nr_pages--) {
1659                 pte = pfn_to_dma_pte(domain, iov_pfn);
1660                 if (!pte)
1661                         return -ENOMEM;
1662                 /* We don't need lock here, nobody else
1663                  * touches the iova range
1664                  */
1665                 BUG_ON(dma_pte_addr(pte));
1666                 dma_set_pte_pfn(pte, phys_pfn);
1667                 dma_set_pte_prot(pte, prot);
1668                 if (prot & DMA_PTE_SNP)
1669                         dma_set_pte_snp(pte);
1670                 domain_flush_cache(domain, pte, sizeof(*pte));
1671                 iov_pfn++;
1672                 phys_pfn++;
1673         }
1674         return 0;
1675 }
1676
1677 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
1678 {
1679         if (!iommu)
1680                 return;
1681
1682         clear_context_table(iommu, bus, devfn);
1683         iommu->flush.flush_context(iommu, 0, 0, 0,
1684                                            DMA_CCMD_GLOBAL_INVL);
1685         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
1686 }
1687
1688 static void domain_remove_dev_info(struct dmar_domain *domain)
1689 {
1690         struct device_domain_info *info;
1691         unsigned long flags;
1692         struct intel_iommu *iommu;
1693
1694         spin_lock_irqsave(&device_domain_lock, flags);
1695         while (!list_empty(&domain->devices)) {
1696                 info = list_entry(domain->devices.next,
1697                         struct device_domain_info, link);
1698                 list_del(&info->link);
1699                 list_del(&info->global);
1700                 if (info->dev)
1701                         info->dev->dev.archdata.iommu = NULL;
1702                 spin_unlock_irqrestore(&device_domain_lock, flags);
1703
1704                 iommu_disable_dev_iotlb(info);
1705                 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
1706                 iommu_detach_dev(iommu, info->bus, info->devfn);
1707                 free_devinfo_mem(info);
1708
1709                 spin_lock_irqsave(&device_domain_lock, flags);
1710         }
1711         spin_unlock_irqrestore(&device_domain_lock, flags);
1712 }
1713
1714 /*
1715  * find_domain
1716  * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1717  */
1718 static struct dmar_domain *
1719 find_domain(struct pci_dev *pdev)
1720 {
1721         struct device_domain_info *info;
1722
1723         /* No lock here, assumes no domain exit in normal case */
1724         info = pdev->dev.archdata.iommu;
1725         if (info)
1726                 return info->domain;
1727         return NULL;
1728 }
1729
1730 /* domain is initialized */
1731 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1732 {
1733         struct dmar_domain *domain, *found = NULL;
1734         struct intel_iommu *iommu;
1735         struct dmar_drhd_unit *drhd;
1736         struct device_domain_info *info, *tmp;
1737         struct pci_dev *dev_tmp;
1738         unsigned long flags;
1739         int bus = 0, devfn = 0;
1740         int segment;
1741         int ret;
1742
1743         domain = find_domain(pdev);
1744         if (domain)
1745                 return domain;
1746
1747         segment = pci_domain_nr(pdev->bus);
1748
1749         dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1750         if (dev_tmp) {
1751                 if (dev_tmp->is_pcie) {
1752                         bus = dev_tmp->subordinate->number;
1753                         devfn = 0;
1754                 } else {
1755                         bus = dev_tmp->bus->number;
1756                         devfn = dev_tmp->devfn;
1757                 }
1758                 spin_lock_irqsave(&device_domain_lock, flags);
1759                 list_for_each_entry(info, &device_domain_list, global) {
1760                         if (info->segment == segment &&
1761                             info->bus == bus && info->devfn == devfn) {
1762                                 found = info->domain;
1763                                 break;
1764                         }
1765                 }
1766                 spin_unlock_irqrestore(&device_domain_lock, flags);
1767                 /* pcie-pci bridge already has a domain, uses it */
1768                 if (found) {
1769                         domain = found;
1770                         goto found_domain;
1771                 }
1772         }
1773
1774         domain = alloc_domain();
1775         if (!domain)
1776                 goto error;
1777
1778         /* Allocate new domain for the device */
1779         drhd = dmar_find_matched_drhd_unit(pdev);
1780         if (!drhd) {
1781                 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1782                         pci_name(pdev));
1783                 return NULL;
1784         }
1785         iommu = drhd->iommu;
1786
1787         ret = iommu_attach_domain(domain, iommu);
1788         if (ret) {
1789                 domain_exit(domain);
1790                 goto error;
1791         }
1792
1793         if (domain_init(domain, gaw)) {
1794                 domain_exit(domain);
1795                 goto error;
1796         }
1797
1798         /* register pcie-to-pci device */
1799         if (dev_tmp) {
1800                 info = alloc_devinfo_mem();
1801                 if (!info) {
1802                         domain_exit(domain);
1803                         goto error;
1804                 }
1805                 info->segment = segment;
1806                 info->bus = bus;
1807                 info->devfn = devfn;
1808                 info->dev = NULL;
1809                 info->domain = domain;
1810                 /* This domain is shared by devices under p2p bridge */
1811                 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
1812
1813                 /* pcie-to-pci bridge already has a domain, uses it */
1814                 found = NULL;
1815                 spin_lock_irqsave(&device_domain_lock, flags);
1816                 list_for_each_entry(tmp, &device_domain_list, global) {
1817                         if (tmp->segment == segment &&
1818                             tmp->bus == bus && tmp->devfn == devfn) {
1819                                 found = tmp->domain;
1820                                 break;
1821                         }
1822                 }
1823                 if (found) {
1824                         free_devinfo_mem(info);
1825                         domain_exit(domain);
1826                         domain = found;
1827                 } else {
1828                         list_add(&info->link, &domain->devices);
1829                         list_add(&info->global, &device_domain_list);
1830                 }
1831                 spin_unlock_irqrestore(&device_domain_lock, flags);
1832         }
1833
1834 found_domain:
1835         info = alloc_devinfo_mem();
1836         if (!info)
1837                 goto error;
1838         info->segment = segment;
1839         info->bus = pdev->bus->number;
1840         info->devfn = pdev->devfn;
1841         info->dev = pdev;
1842         info->domain = domain;
1843         spin_lock_irqsave(&device_domain_lock, flags);
1844         /* somebody is fast */
1845         found = find_domain(pdev);
1846         if (found != NULL) {
1847                 spin_unlock_irqrestore(&device_domain_lock, flags);
1848                 if (found != domain) {
1849                         domain_exit(domain);
1850                         domain = found;
1851                 }
1852                 free_devinfo_mem(info);
1853                 return domain;
1854         }
1855         list_add(&info->link, &domain->devices);
1856         list_add(&info->global, &device_domain_list);
1857         pdev->dev.archdata.iommu = info;
1858         spin_unlock_irqrestore(&device_domain_lock, flags);
1859         return domain;
1860 error:
1861         /* recheck it here, maybe others set it */
1862         return find_domain(pdev);
1863 }
1864
1865 static int iommu_identity_mapping;
1866
1867 static int iommu_domain_identity_map(struct dmar_domain *domain,
1868                                      unsigned long long start,
1869                                      unsigned long long end)
1870 {
1871         unsigned long size;
1872         unsigned long long base;
1873
1874         /* The address might not be aligned */
1875         base = start & PAGE_MASK;
1876         size = end - base;
1877         size = PAGE_ALIGN(size);
1878         if (!reserve_iova(&domain->iovad, IOVA_PFN(base),
1879                         IOVA_PFN(base + size) - 1)) {
1880                 printk(KERN_ERR "IOMMU: reserve iova failed\n");
1881                 return -ENOMEM;
1882         }
1883
1884         pr_debug("Mapping reserved region %lx@%llx for domain %d\n",
1885                  size, base, domain->id);
1886         /*
1887          * RMRR range might have overlap with physical memory range,
1888          * clear it first
1889          */
1890         dma_pte_clear_range(domain, base >> VTD_PAGE_SHIFT,
1891                             (base + size - 1) >> VTD_PAGE_SHIFT);
1892
1893         return domain_pfn_mapping(domain, base >> VTD_PAGE_SHIFT,
1894                                   base >> VTD_PAGE_SHIFT,
1895                                   size >> VTD_PAGE_SHIFT,
1896                                   DMA_PTE_READ|DMA_PTE_WRITE);
1897 }
1898
1899 static int iommu_prepare_identity_map(struct pci_dev *pdev,
1900                                       unsigned long long start,
1901                                       unsigned long long end)
1902 {
1903         struct dmar_domain *domain;
1904         int ret;
1905
1906         printk(KERN_INFO
1907                "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1908                pci_name(pdev), start, end);
1909
1910         domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1911         if (!domain)
1912                 return -ENOMEM;
1913
1914         ret = iommu_domain_identity_map(domain, start, end);
1915         if (ret)
1916                 goto error;
1917
1918         /* context entry init */
1919         ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
1920         if (ret)
1921                 goto error;
1922
1923         return 0;
1924
1925  error:
1926         domain_exit(domain);
1927         return ret;
1928 }
1929
1930 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
1931         struct pci_dev *pdev)
1932 {
1933         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
1934                 return 0;
1935         return iommu_prepare_identity_map(pdev, rmrr->base_address,
1936                 rmrr->end_address + 1);
1937 }
1938
1939 #ifdef CONFIG_DMAR_FLOPPY_WA
1940 static inline void iommu_prepare_isa(void)
1941 {
1942         struct pci_dev *pdev;
1943         int ret;
1944
1945         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
1946         if (!pdev)
1947                 return;
1948
1949         printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
1950         ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
1951
1952         if (ret)
1953                 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
1954                        "floppy might not work\n");
1955
1956 }
1957 #else
1958 static inline void iommu_prepare_isa(void)
1959 {
1960         return;
1961 }
1962 #endif /* !CONFIG_DMAR_FLPY_WA */
1963
1964 /* Initialize each context entry as pass through.*/
1965 static int __init init_context_pass_through(void)
1966 {
1967         struct pci_dev *pdev = NULL;
1968         struct dmar_domain *domain;
1969         int ret;
1970
1971         for_each_pci_dev(pdev) {
1972                 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1973                 ret = domain_context_mapping(domain, pdev,
1974                                              CONTEXT_TT_PASS_THROUGH);
1975                 if (ret)
1976                         return ret;
1977         }
1978         return 0;
1979 }
1980
1981 static int md_domain_init(struct dmar_domain *domain, int guest_width);
1982
1983 static int __init si_domain_work_fn(unsigned long start_pfn,
1984                                     unsigned long end_pfn, void *datax)
1985 {
1986         int *ret = datax;
1987
1988         *ret = iommu_domain_identity_map(si_domain,
1989                                          (uint64_t)start_pfn << PAGE_SHIFT,
1990                                          (uint64_t)end_pfn << PAGE_SHIFT);
1991         return *ret;
1992
1993 }
1994
1995 static int si_domain_init(void)
1996 {
1997         struct dmar_drhd_unit *drhd;
1998         struct intel_iommu *iommu;
1999         int nid, ret = 0;
2000
2001         si_domain = alloc_domain();
2002         if (!si_domain)
2003                 return -EFAULT;
2004
2005         pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
2006
2007         for_each_active_iommu(iommu, drhd) {
2008                 ret = iommu_attach_domain(si_domain, iommu);
2009                 if (ret) {
2010                         domain_exit(si_domain);
2011                         return -EFAULT;
2012                 }
2013         }
2014
2015         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2016                 domain_exit(si_domain);
2017                 return -EFAULT;
2018         }
2019
2020         si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2021
2022         for_each_online_node(nid) {
2023                 work_with_active_regions(nid, si_domain_work_fn, &ret);
2024                 if (ret)
2025                         return ret;
2026         }
2027
2028         return 0;
2029 }
2030
2031 static void domain_remove_one_dev_info(struct dmar_domain *domain,
2032                                           struct pci_dev *pdev);
2033 static int identity_mapping(struct pci_dev *pdev)
2034 {
2035         struct device_domain_info *info;
2036
2037         if (likely(!iommu_identity_mapping))
2038                 return 0;
2039
2040
2041         list_for_each_entry(info, &si_domain->devices, link)
2042                 if (info->dev == pdev)
2043                         return 1;
2044         return 0;
2045 }
2046
2047 static int domain_add_dev_info(struct dmar_domain *domain,
2048                                   struct pci_dev *pdev)
2049 {
2050         struct device_domain_info *info;
2051         unsigned long flags;
2052
2053         info = alloc_devinfo_mem();
2054         if (!info)
2055                 return -ENOMEM;
2056
2057         info->segment = pci_domain_nr(pdev->bus);
2058         info->bus = pdev->bus->number;
2059         info->devfn = pdev->devfn;
2060         info->dev = pdev;
2061         info->domain = domain;
2062
2063         spin_lock_irqsave(&device_domain_lock, flags);
2064         list_add(&info->link, &domain->devices);
2065         list_add(&info->global, &device_domain_list);
2066         pdev->dev.archdata.iommu = info;
2067         spin_unlock_irqrestore(&device_domain_lock, flags);
2068
2069         return 0;
2070 }
2071
2072 static int iommu_prepare_static_identity_mapping(void)
2073 {
2074         struct pci_dev *pdev = NULL;
2075         int ret;
2076
2077         ret = si_domain_init();
2078         if (ret)
2079                 return -EFAULT;
2080
2081         for_each_pci_dev(pdev) {
2082                 printk(KERN_INFO "IOMMU: identity mapping for device %s\n",
2083                        pci_name(pdev));
2084
2085                 ret = domain_context_mapping(si_domain, pdev,
2086                                              CONTEXT_TT_MULTI_LEVEL);
2087                 if (ret)
2088                         return ret;
2089                 ret = domain_add_dev_info(si_domain, pdev);
2090                 if (ret)
2091                         return ret;
2092         }
2093
2094         return 0;
2095 }
2096
2097 int __init init_dmars(void)
2098 {
2099         struct dmar_drhd_unit *drhd;
2100         struct dmar_rmrr_unit *rmrr;
2101         struct pci_dev *pdev;
2102         struct intel_iommu *iommu;
2103         int i, ret;
2104         int pass_through = 1;
2105
2106         /*
2107          * In case pass through can not be enabled, iommu tries to use identity
2108          * mapping.
2109          */
2110         if (iommu_pass_through)
2111                 iommu_identity_mapping = 1;
2112
2113         /*
2114          * for each drhd
2115          *    allocate root
2116          *    initialize and program root entry to not present
2117          * endfor
2118          */
2119         for_each_drhd_unit(drhd) {
2120                 g_num_of_iommus++;
2121                 /*
2122                  * lock not needed as this is only incremented in the single
2123                  * threaded kernel __init code path all other access are read
2124                  * only
2125                  */
2126         }
2127
2128         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2129                         GFP_KERNEL);
2130         if (!g_iommus) {
2131                 printk(KERN_ERR "Allocating global iommu array failed\n");
2132                 ret = -ENOMEM;
2133                 goto error;
2134         }
2135
2136         deferred_flush = kzalloc(g_num_of_iommus *
2137                 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2138         if (!deferred_flush) {
2139                 kfree(g_iommus);
2140                 ret = -ENOMEM;
2141                 goto error;
2142         }
2143
2144         for_each_drhd_unit(drhd) {
2145                 if (drhd->ignored)
2146                         continue;
2147
2148                 iommu = drhd->iommu;
2149                 g_iommus[iommu->seq_id] = iommu;
2150
2151                 ret = iommu_init_domains(iommu);
2152                 if (ret)
2153                         goto error;
2154
2155                 /*
2156                  * TBD:
2157                  * we could share the same root & context tables
2158                  * amoung all IOMMU's. Need to Split it later.
2159                  */
2160                 ret = iommu_alloc_root_entry(iommu);
2161                 if (ret) {
2162                         printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2163                         goto error;
2164                 }
2165                 if (!ecap_pass_through(iommu->ecap))
2166                         pass_through = 0;
2167         }
2168         if (iommu_pass_through)
2169                 if (!pass_through) {
2170                         printk(KERN_INFO
2171                                "Pass Through is not supported by hardware.\n");
2172                         iommu_pass_through = 0;
2173                 }
2174
2175         /*
2176          * Start from the sane iommu hardware state.
2177          */
2178         for_each_drhd_unit(drhd) {
2179                 if (drhd->ignored)
2180                         continue;
2181
2182                 iommu = drhd->iommu;
2183
2184                 /*
2185                  * If the queued invalidation is already initialized by us
2186                  * (for example, while enabling interrupt-remapping) then
2187                  * we got the things already rolling from a sane state.
2188                  */
2189                 if (iommu->qi)
2190                         continue;
2191
2192                 /*
2193                  * Clear any previous faults.
2194                  */
2195                 dmar_fault(-1, iommu);
2196                 /*
2197                  * Disable queued invalidation if supported and already enabled
2198                  * before OS handover.
2199                  */
2200                 dmar_disable_qi(iommu);
2201         }
2202
2203         for_each_drhd_unit(drhd) {
2204                 if (drhd->ignored)
2205                         continue;
2206
2207                 iommu = drhd->iommu;
2208
2209                 if (dmar_enable_qi(iommu)) {
2210                         /*
2211                          * Queued Invalidate not enabled, use Register Based
2212                          * Invalidate
2213                          */
2214                         iommu->flush.flush_context = __iommu_flush_context;
2215                         iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2216                         printk(KERN_INFO "IOMMU 0x%Lx: using Register based "
2217                                "invalidation\n",
2218                                (unsigned long long)drhd->reg_base_addr);
2219                 } else {
2220                         iommu->flush.flush_context = qi_flush_context;
2221                         iommu->flush.flush_iotlb = qi_flush_iotlb;
2222                         printk(KERN_INFO "IOMMU 0x%Lx: using Queued "
2223                                "invalidation\n",
2224                                (unsigned long long)drhd->reg_base_addr);
2225                 }
2226         }
2227
2228         /*
2229          * If pass through is set and enabled, context entries of all pci
2230          * devices are intialized by pass through translation type.
2231          */
2232         if (iommu_pass_through) {
2233                 ret = init_context_pass_through();
2234                 if (ret) {
2235                         printk(KERN_ERR "IOMMU: Pass through init failed.\n");
2236                         iommu_pass_through = 0;
2237                 }
2238         }
2239
2240         /*
2241          * If pass through is not set or not enabled, setup context entries for
2242          * identity mappings for rmrr, gfx, and isa and may fall back to static
2243          * identity mapping if iommu_identity_mapping is set.
2244          */
2245         if (!iommu_pass_through) {
2246                 if (iommu_identity_mapping)
2247                         iommu_prepare_static_identity_mapping();
2248                 /*
2249                  * For each rmrr
2250                  *   for each dev attached to rmrr
2251                  *   do
2252                  *     locate drhd for dev, alloc domain for dev
2253                  *     allocate free domain
2254                  *     allocate page table entries for rmrr
2255                  *     if context not allocated for bus
2256                  *           allocate and init context
2257                  *           set present in root table for this bus
2258                  *     init context with domain, translation etc
2259                  *    endfor
2260                  * endfor
2261                  */
2262                 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2263                 for_each_rmrr_units(rmrr) {
2264                         for (i = 0; i < rmrr->devices_cnt; i++) {
2265                                 pdev = rmrr->devices[i];
2266                                 /*
2267                                  * some BIOS lists non-exist devices in DMAR
2268                                  * table.
2269                                  */
2270                                 if (!pdev)
2271                                         continue;
2272                                 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2273                                 if (ret)
2274                                         printk(KERN_ERR
2275                                  "IOMMU: mapping reserved region failed\n");
2276                         }
2277                 }
2278
2279                 iommu_prepare_isa();
2280         }
2281
2282         /*
2283          * for each drhd
2284          *   enable fault log
2285          *   global invalidate context cache
2286          *   global invalidate iotlb
2287          *   enable translation
2288          */
2289         for_each_drhd_unit(drhd) {
2290                 if (drhd->ignored)
2291                         continue;
2292                 iommu = drhd->iommu;
2293
2294                 iommu_flush_write_buffer(iommu);
2295
2296                 ret = dmar_set_interrupt(iommu);
2297                 if (ret)
2298                         goto error;
2299
2300                 iommu_set_root_entry(iommu);
2301
2302                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
2303                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2304                 iommu_disable_protect_mem_regions(iommu);
2305
2306                 ret = iommu_enable_translation(iommu);
2307                 if (ret)
2308                         goto error;
2309         }
2310
2311         return 0;
2312 error:
2313         for_each_drhd_unit(drhd) {
2314                 if (drhd->ignored)
2315                         continue;
2316                 iommu = drhd->iommu;
2317                 free_iommu(iommu);
2318         }
2319         kfree(g_iommus);
2320         return ret;
2321 }
2322
2323 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2324                                             size_t size)
2325 {
2326         host_addr &= ~PAGE_MASK;
2327         host_addr += size + PAGE_SIZE - 1;
2328
2329         return host_addr >> VTD_PAGE_SHIFT;
2330 }
2331
2332 struct iova *
2333 iommu_alloc_iova(struct dmar_domain *domain, size_t size, u64 end)
2334 {
2335         struct iova *piova;
2336
2337         /* Make sure it's in range */
2338         end = min_t(u64, DOMAIN_MAX_ADDR(domain->gaw), end);
2339         if (!size || (IOVA_START_ADDR + size > end))
2340                 return NULL;
2341
2342         piova = alloc_iova(&domain->iovad,
2343                         size >> PAGE_SHIFT, IOVA_PFN(end), 1);
2344         return piova;
2345 }
2346
2347 static struct iova *
2348 __intel_alloc_iova(struct device *dev, struct dmar_domain *domain,
2349                    size_t size, u64 dma_mask)
2350 {
2351         struct pci_dev *pdev = to_pci_dev(dev);
2352         struct iova *iova = NULL;
2353
2354         if (dma_mask <= DMA_BIT_MASK(32) || dmar_forcedac)
2355                 iova = iommu_alloc_iova(domain, size, dma_mask);
2356         else {
2357                 /*
2358                  * First try to allocate an io virtual address in
2359                  * DMA_BIT_MASK(32) and if that fails then try allocating
2360                  * from higher range
2361                  */
2362                 iova = iommu_alloc_iova(domain, size, DMA_BIT_MASK(32));
2363                 if (!iova)
2364                         iova = iommu_alloc_iova(domain, size, dma_mask);
2365         }
2366
2367         if (!iova) {
2368                 printk(KERN_ERR"Allocating iova for %s failed", pci_name(pdev));
2369                 return NULL;
2370         }
2371
2372         return iova;
2373 }
2374
2375 static struct dmar_domain *
2376 get_valid_domain_for_dev(struct pci_dev *pdev)
2377 {
2378         struct dmar_domain *domain;
2379         int ret;
2380
2381         domain = get_domain_for_dev(pdev,
2382                         DEFAULT_DOMAIN_ADDRESS_WIDTH);
2383         if (!domain) {
2384                 printk(KERN_ERR
2385                         "Allocating domain for %s failed", pci_name(pdev));
2386                 return NULL;
2387         }
2388
2389         /* make sure context mapping is ok */
2390         if (unlikely(!domain_context_mapped(pdev))) {
2391                 ret = domain_context_mapping(domain, pdev,
2392                                              CONTEXT_TT_MULTI_LEVEL);
2393                 if (ret) {
2394                         printk(KERN_ERR
2395                                 "Domain context map for %s failed",
2396                                 pci_name(pdev));
2397                         return NULL;
2398                 }
2399         }
2400
2401         return domain;
2402 }
2403
2404 static int iommu_dummy(struct pci_dev *pdev)
2405 {
2406         return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2407 }
2408
2409 /* Check if the pdev needs to go through non-identity map and unmap process.*/
2410 static int iommu_no_mapping(struct pci_dev *pdev)
2411 {
2412         int found;
2413
2414         if (!iommu_identity_mapping)
2415                 return iommu_dummy(pdev);
2416
2417         found = identity_mapping(pdev);
2418         if (found) {
2419                 if (pdev->dma_mask > DMA_BIT_MASK(32))
2420                         return 1;
2421                 else {
2422                         /*
2423                          * 32 bit DMA is removed from si_domain and fall back
2424                          * to non-identity mapping.
2425                          */
2426                         domain_remove_one_dev_info(si_domain, pdev);
2427                         printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2428                                pci_name(pdev));
2429                         return 0;
2430                 }
2431         } else {
2432                 /*
2433                  * In case of a detached 64 bit DMA device from vm, the device
2434                  * is put into si_domain for identity mapping.
2435                  */
2436                 if (pdev->dma_mask > DMA_BIT_MASK(32)) {
2437                         int ret;
2438                         ret = domain_add_dev_info(si_domain, pdev);
2439                         if (!ret) {
2440                                 printk(KERN_INFO "64bit %s uses identity mapping\n",
2441                                        pci_name(pdev));
2442                                 return 1;
2443                         }
2444                 }
2445         }
2446
2447         return iommu_dummy(pdev);
2448 }
2449
2450 static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2451                                      size_t size, int dir, u64 dma_mask)
2452 {
2453         struct pci_dev *pdev = to_pci_dev(hwdev);
2454         struct dmar_domain *domain;
2455         phys_addr_t start_paddr;
2456         struct iova *iova;
2457         int prot = 0;
2458         int ret;
2459         struct intel_iommu *iommu;
2460
2461         BUG_ON(dir == DMA_NONE);
2462
2463         if (iommu_no_mapping(pdev))
2464                 return paddr;
2465
2466         domain = get_valid_domain_for_dev(pdev);
2467         if (!domain)
2468                 return 0;
2469
2470         iommu = domain_get_iommu(domain);
2471         size = aligned_nrpages(paddr, size);
2472
2473         iova = __intel_alloc_iova(hwdev, domain, size << VTD_PAGE_SHIFT, pdev->dma_mask);
2474         if (!iova)
2475                 goto error;
2476
2477         /*
2478          * Check if DMAR supports zero-length reads on write only
2479          * mappings..
2480          */
2481         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2482                         !cap_zlr(iommu->cap))
2483                 prot |= DMA_PTE_READ;
2484         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2485                 prot |= DMA_PTE_WRITE;
2486         /*
2487          * paddr - (paddr + size) might be partial page, we should map the whole
2488          * page.  Note: if two part of one page are separately mapped, we
2489          * might have two guest_addr mapping to the same host paddr, but this
2490          * is not a big problem
2491          */
2492         ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
2493                                  paddr >> VTD_PAGE_SHIFT, size, prot);
2494         if (ret)
2495                 goto error;
2496
2497         /* it's a non-present to present mapping. Only flush if caching mode */
2498         if (cap_caching_mode(iommu->cap))
2499                 iommu_flush_iotlb_psi(iommu, 0, mm_to_dma_pfn(iova->pfn_lo), size);
2500         else
2501                 iommu_flush_write_buffer(iommu);
2502
2503         start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2504         start_paddr += paddr & ~PAGE_MASK;
2505         return start_paddr;
2506
2507 error:
2508         if (iova)
2509                 __free_iova(&domain->iovad, iova);
2510         printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
2511                 pci_name(pdev), size, (unsigned long long)paddr, dir);
2512         return 0;
2513 }
2514
2515 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2516                                  unsigned long offset, size_t size,
2517                                  enum dma_data_direction dir,
2518                                  struct dma_attrs *attrs)
2519 {
2520         return __intel_map_single(dev, page_to_phys(page) + offset, size,
2521                                   dir, to_pci_dev(dev)->dma_mask);
2522 }
2523
2524 static void flush_unmaps(void)
2525 {
2526         int i, j;
2527
2528         timer_on = 0;
2529
2530         /* just flush them all */
2531         for (i = 0; i < g_num_of_iommus; i++) {
2532                 struct intel_iommu *iommu = g_iommus[i];
2533                 if (!iommu)
2534                         continue;
2535
2536                 if (!deferred_flush[i].next)
2537                         continue;
2538
2539                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2540                                          DMA_TLB_GLOBAL_FLUSH);
2541                 for (j = 0; j < deferred_flush[i].next; j++) {
2542                         unsigned long mask;
2543                         struct iova *iova = deferred_flush[i].iova[j];
2544
2545                         mask = (iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT;
2546                         mask = ilog2(mask >> VTD_PAGE_SHIFT);
2547                         iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
2548                                         iova->pfn_lo << PAGE_SHIFT, mask);
2549                         __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
2550                 }
2551                 deferred_flush[i].next = 0;
2552         }
2553
2554         list_size = 0;
2555 }
2556
2557 static void flush_unmaps_timeout(unsigned long data)
2558 {
2559         unsigned long flags;
2560
2561         spin_lock_irqsave(&async_umap_flush_lock, flags);
2562         flush_unmaps();
2563         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2564 }
2565
2566 static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2567 {
2568         unsigned long flags;
2569         int next, iommu_id;
2570         struct intel_iommu *iommu;
2571
2572         spin_lock_irqsave(&async_umap_flush_lock, flags);
2573         if (list_size == HIGH_WATER_MARK)
2574                 flush_unmaps();
2575
2576         iommu = domain_get_iommu(dom);
2577         iommu_id = iommu->seq_id;
2578
2579         next = deferred_flush[iommu_id].next;
2580         deferred_flush[iommu_id].domain[next] = dom;
2581         deferred_flush[iommu_id].iova[next] = iova;
2582         deferred_flush[iommu_id].next++;
2583
2584         if (!timer_on) {
2585                 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2586                 timer_on = 1;
2587         }
2588         list_size++;
2589         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2590 }
2591
2592 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2593                              size_t size, enum dma_data_direction dir,
2594                              struct dma_attrs *attrs)
2595 {
2596         struct pci_dev *pdev = to_pci_dev(dev);
2597         struct dmar_domain *domain;
2598         unsigned long start_pfn, last_pfn;
2599         struct iova *iova;
2600         struct intel_iommu *iommu;
2601
2602         if (iommu_no_mapping(pdev))
2603                 return;
2604
2605         domain = find_domain(pdev);
2606         BUG_ON(!domain);
2607
2608         iommu = domain_get_iommu(domain);
2609
2610         iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
2611         if (!iova)
2612                 return;
2613
2614         start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2615         last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2616
2617         pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2618                  pci_name(pdev), start_pfn, last_pfn);
2619
2620         /*  clear the whole page */
2621         dma_pte_clear_range(domain, start_pfn, last_pfn);
2622
2623         /* free page tables */
2624         dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2625
2626         if (intel_iommu_strict) {
2627                 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2628                                       last_pfn - start_pfn + 1);
2629                 /* free iova */
2630                 __free_iova(&domain->iovad, iova);
2631         } else {
2632                 add_unmap(domain, iova);
2633                 /*
2634                  * queue up the release of the unmap to save the 1/6th of the
2635                  * cpu used up by the iotlb flush operation...
2636                  */
2637         }
2638 }
2639
2640 static void intel_unmap_single(struct device *dev, dma_addr_t dev_addr, size_t size,
2641                                int dir)
2642 {
2643         intel_unmap_page(dev, dev_addr, size, dir, NULL);
2644 }
2645
2646 static void *intel_alloc_coherent(struct device *hwdev, size_t size,
2647                                   dma_addr_t *dma_handle, gfp_t flags)
2648 {
2649         void *vaddr;
2650         int order;
2651
2652         size = PAGE_ALIGN(size);
2653         order = get_order(size);
2654         flags &= ~(GFP_DMA | GFP_DMA32);
2655
2656         vaddr = (void *)__get_free_pages(flags, order);
2657         if (!vaddr)
2658                 return NULL;
2659         memset(vaddr, 0, size);
2660
2661         *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2662                                          DMA_BIDIRECTIONAL,
2663                                          hwdev->coherent_dma_mask);
2664         if (*dma_handle)
2665                 return vaddr;
2666         free_pages((unsigned long)vaddr, order);
2667         return NULL;
2668 }
2669
2670 static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2671                                 dma_addr_t dma_handle)
2672 {
2673         int order;
2674
2675         size = PAGE_ALIGN(size);
2676         order = get_order(size);
2677
2678         intel_unmap_single(hwdev, dma_handle, size, DMA_BIDIRECTIONAL);
2679         free_pages((unsigned long)vaddr, order);
2680 }
2681
2682 static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2683                            int nelems, enum dma_data_direction dir,
2684                            struct dma_attrs *attrs)
2685 {
2686         struct pci_dev *pdev = to_pci_dev(hwdev);
2687         struct dmar_domain *domain;
2688         unsigned long start_pfn, last_pfn;
2689         struct iova *iova;
2690         struct intel_iommu *iommu;
2691
2692         if (iommu_no_mapping(pdev))
2693                 return;
2694
2695         domain = find_domain(pdev);
2696         BUG_ON(!domain);
2697
2698         iommu = domain_get_iommu(domain);
2699
2700         iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
2701         if (!iova)
2702                 return;
2703
2704         start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2705         last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2706
2707         /*  clear the whole page */
2708         dma_pte_clear_range(domain, start_pfn, last_pfn);
2709
2710         /* free page tables */
2711         dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2712
2713         iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2714                               (last_pfn - start_pfn + 1));
2715
2716         /* free iova */
2717         __free_iova(&domain->iovad, iova);
2718 }
2719
2720 static int intel_nontranslate_map_sg(struct device *hddev,
2721         struct scatterlist *sglist, int nelems, int dir)
2722 {
2723         int i;
2724         struct scatterlist *sg;
2725
2726         for_each_sg(sglist, sg, nelems, i) {
2727                 BUG_ON(!sg_page(sg));
2728                 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
2729                 sg->dma_length = sg->length;
2730         }
2731         return nelems;
2732 }
2733
2734 static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
2735                         enum dma_data_direction dir, struct dma_attrs *attrs)
2736 {
2737         int i;
2738         struct pci_dev *pdev = to_pci_dev(hwdev);
2739         struct dmar_domain *domain;
2740         size_t size = 0;
2741         int prot = 0;
2742         size_t offset_pfn = 0;
2743         struct iova *iova = NULL;
2744         int ret;
2745         struct scatterlist *sg;
2746         unsigned long start_vpfn;
2747         struct intel_iommu *iommu;
2748
2749         BUG_ON(dir == DMA_NONE);
2750         if (iommu_no_mapping(pdev))
2751                 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
2752
2753         domain = get_valid_domain_for_dev(pdev);
2754         if (!domain)
2755                 return 0;
2756
2757         iommu = domain_get_iommu(domain);
2758
2759         for_each_sg(sglist, sg, nelems, i)
2760                 size += aligned_nrpages(sg->offset, sg->length);
2761
2762         iova = __intel_alloc_iova(hwdev, domain, size << VTD_PAGE_SHIFT,
2763                                   pdev->dma_mask);
2764         if (!iova) {
2765                 sglist->dma_length = 0;
2766                 return 0;
2767         }
2768
2769         /*
2770          * Check if DMAR supports zero-length reads on write only
2771          * mappings..
2772          */
2773         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2774                         !cap_zlr(iommu->cap))
2775                 prot |= DMA_PTE_READ;
2776         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2777                 prot |= DMA_PTE_WRITE;
2778
2779         start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
2780         offset_pfn = 0;
2781         for_each_sg(sglist, sg, nelems, i) {
2782                 int nr_pages = aligned_nrpages(sg->offset, sg->length);
2783                 ret = domain_pfn_mapping(domain, start_vpfn + offset_pfn,
2784                                          page_to_dma_pfn(sg_page(sg)),
2785                                          nr_pages, prot);
2786                 if (ret) {
2787                         /*  clear the page */
2788                         dma_pte_clear_range(domain, start_vpfn,
2789                                             start_vpfn + offset_pfn);
2790                         /* free page tables */
2791                         dma_pte_free_pagetable(domain, start_vpfn,
2792                                                start_vpfn + offset_pfn);
2793                         /* free iova */
2794                         __free_iova(&domain->iovad, iova);
2795                         return 0;
2796                 }
2797                 sg->dma_address = ((dma_addr_t)(start_vpfn + offset_pfn)
2798                                    << VTD_PAGE_SHIFT) + sg->offset;
2799                 sg->dma_length = sg->length;
2800                 offset_pfn += nr_pages;
2801         }
2802
2803         /* it's a non-present to present mapping. Only flush if caching mode */
2804         if (cap_caching_mode(iommu->cap))
2805                 iommu_flush_iotlb_psi(iommu, 0, start_vpfn, offset_pfn);
2806         else
2807                 iommu_flush_write_buffer(iommu);
2808
2809         return nelems;
2810 }
2811
2812 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
2813 {
2814         return !dma_addr;
2815 }
2816
2817 struct dma_map_ops intel_dma_ops = {
2818         .alloc_coherent = intel_alloc_coherent,
2819         .free_coherent = intel_free_coherent,
2820         .map_sg = intel_map_sg,
2821         .unmap_sg = intel_unmap_sg,
2822         .map_page = intel_map_page,
2823         .unmap_page = intel_unmap_page,
2824         .mapping_error = intel_mapping_error,
2825 };
2826
2827 static inline int iommu_domain_cache_init(void)
2828 {
2829         int ret = 0;
2830
2831         iommu_domain_cache = kmem_cache_create("iommu_domain",
2832                                          sizeof(struct dmar_domain),
2833                                          0,
2834                                          SLAB_HWCACHE_ALIGN,
2835
2836                                          NULL);
2837         if (!iommu_domain_cache) {
2838                 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
2839                 ret = -ENOMEM;
2840         }
2841
2842         return ret;
2843 }
2844
2845 static inline int iommu_devinfo_cache_init(void)
2846 {
2847         int ret = 0;
2848
2849         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
2850                                          sizeof(struct device_domain_info),
2851                                          0,
2852                                          SLAB_HWCACHE_ALIGN,
2853                                          NULL);
2854         if (!iommu_devinfo_cache) {
2855                 printk(KERN_ERR "Couldn't create devinfo cache\n");
2856                 ret = -ENOMEM;
2857         }
2858
2859         return ret;
2860 }
2861
2862 static inline int iommu_iova_cache_init(void)
2863 {
2864         int ret = 0;
2865
2866         iommu_iova_cache = kmem_cache_create("iommu_iova",
2867                                          sizeof(struct iova),
2868                                          0,
2869                                          SLAB_HWCACHE_ALIGN,
2870                                          NULL);
2871         if (!iommu_iova_cache) {
2872                 printk(KERN_ERR "Couldn't create iova cache\n");
2873                 ret = -ENOMEM;
2874         }
2875
2876         return ret;
2877 }
2878
2879 static int __init iommu_init_mempool(void)
2880 {
2881         int ret;
2882         ret = iommu_iova_cache_init();
2883         if (ret)
2884                 return ret;
2885
2886         ret = iommu_domain_cache_init();
2887         if (ret)
2888                 goto domain_error;
2889
2890         ret = iommu_devinfo_cache_init();
2891         if (!ret)
2892                 return ret;
2893
2894         kmem_cache_destroy(iommu_domain_cache);
2895 domain_error:
2896         kmem_cache_destroy(iommu_iova_cache);
2897
2898         return -ENOMEM;
2899 }
2900
2901 static void __init iommu_exit_mempool(void)
2902 {
2903         kmem_cache_destroy(iommu_devinfo_cache);
2904         kmem_cache_destroy(iommu_domain_cache);
2905         kmem_cache_destroy(iommu_iova_cache);
2906
2907 }
2908
2909 static void __init init_no_remapping_devices(void)
2910 {
2911         struct dmar_drhd_unit *drhd;
2912
2913         for_each_drhd_unit(drhd) {
2914                 if (!drhd->include_all) {
2915                         int i;
2916                         for (i = 0; i < drhd->devices_cnt; i++)
2917                                 if (drhd->devices[i] != NULL)
2918                                         break;
2919                         /* ignore DMAR unit if no pci devices exist */
2920                         if (i == drhd->devices_cnt)
2921                                 drhd->ignored = 1;
2922                 }
2923         }
2924
2925         if (dmar_map_gfx)
2926                 return;
2927
2928         for_each_drhd_unit(drhd) {
2929                 int i;
2930                 if (drhd->ignored || drhd->include_all)
2931                         continue;
2932
2933                 for (i = 0; i < drhd->devices_cnt; i++)
2934                         if (drhd->devices[i] &&
2935                                 !IS_GFX_DEVICE(drhd->devices[i]))
2936                                 break;
2937
2938                 if (i < drhd->devices_cnt)
2939                         continue;
2940
2941                 /* bypass IOMMU if it is just for gfx devices */
2942                 drhd->ignored = 1;
2943                 for (i = 0; i < drhd->devices_cnt; i++) {
2944                         if (!drhd->devices[i])
2945                                 continue;
2946                         drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
2947                 }
2948         }
2949 }
2950
2951 #ifdef CONFIG_SUSPEND
2952 static int init_iommu_hw(void)
2953 {
2954         struct dmar_drhd_unit *drhd;
2955         struct intel_iommu *iommu = NULL;
2956
2957         for_each_active_iommu(iommu, drhd)
2958                 if (iommu->qi)
2959                         dmar_reenable_qi(iommu);
2960
2961         for_each_active_iommu(iommu, drhd) {
2962                 iommu_flush_write_buffer(iommu);
2963
2964                 iommu_set_root_entry(iommu);
2965
2966                 iommu->flush.flush_context(iommu, 0, 0, 0,
2967                                            DMA_CCMD_GLOBAL_INVL);
2968                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2969                                          DMA_TLB_GLOBAL_FLUSH);
2970                 iommu_disable_protect_mem_regions(iommu);
2971                 iommu_enable_translation(iommu);
2972         }
2973
2974         return 0;
2975 }
2976
2977 static void iommu_flush_all(void)
2978 {
2979         struct dmar_drhd_unit *drhd;
2980         struct intel_iommu *iommu;
2981
2982         for_each_active_iommu(iommu, drhd) {
2983                 iommu->flush.flush_context(iommu, 0, 0, 0,
2984                                            DMA_CCMD_GLOBAL_INVL);
2985                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2986                                          DMA_TLB_GLOBAL_FLUSH);
2987         }
2988 }
2989
2990 static int iommu_suspend(struct sys_device *dev, pm_message_t state)
2991 {
2992         struct dmar_drhd_unit *drhd;
2993         struct intel_iommu *iommu = NULL;
2994         unsigned long flag;
2995
2996         for_each_active_iommu(iommu, drhd) {
2997                 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
2998                                                  GFP_ATOMIC);
2999                 if (!iommu->iommu_state)
3000                         goto nomem;
3001         }
3002
3003         iommu_flush_all();
3004
3005         for_each_active_iommu(iommu, drhd) {
3006                 iommu_disable_translation(iommu);
3007
3008                 spin_lock_irqsave(&iommu->register_lock, flag);
3009
3010                 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3011                         readl(iommu->reg + DMAR_FECTL_REG);
3012                 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3013                         readl(iommu->reg + DMAR_FEDATA_REG);
3014                 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3015                         readl(iommu->reg + DMAR_FEADDR_REG);
3016                 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3017                         readl(iommu->reg + DMAR_FEUADDR_REG);
3018
3019                 spin_unlock_irqrestore(&iommu->register_lock, flag);
3020         }
3021         return 0;
3022
3023 nomem:
3024         for_each_active_iommu(iommu, drhd)
3025                 kfree(iommu->iommu_state);
3026
3027         return -ENOMEM;
3028 }
3029
3030 static int iommu_resume(struct sys_device *dev)
3031 {
3032         struct dmar_drhd_unit *drhd;
3033         struct intel_iommu *iommu = NULL;
3034         unsigned long flag;
3035
3036         if (init_iommu_hw()) {
3037                 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3038                 return -EIO;
3039         }
3040
3041         for_each_active_iommu(iommu, drhd) {
3042
3043                 spin_lock_irqsave(&iommu->register_lock, flag);
3044
3045                 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3046                         iommu->reg + DMAR_FECTL_REG);
3047                 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3048                         iommu->reg + DMAR_FEDATA_REG);
3049                 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3050                         iommu->reg + DMAR_FEADDR_REG);
3051                 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3052                         iommu->reg + DMAR_FEUADDR_REG);
3053
3054                 spin_unlock_irqrestore(&iommu->register_lock, flag);
3055         }
3056
3057         for_each_active_iommu(iommu, drhd)
3058                 kfree(iommu->iommu_state);
3059
3060         return 0;
3061 }
3062
3063 static struct sysdev_class iommu_sysclass = {
3064         .name           = "iommu",
3065         .resume         = iommu_resume,
3066         .suspend        = iommu_suspend,
3067 };
3068
3069 static struct sys_device device_iommu = {
3070         .cls    = &iommu_sysclass,
3071 };
3072
3073 static int __init init_iommu_sysfs(void)
3074 {
3075         int error;
3076
3077         error = sysdev_class_register(&iommu_sysclass);
3078         if (error)
3079                 return error;
3080
3081         error = sysdev_register(&device_iommu);
3082         if (error)
3083                 sysdev_class_unregister(&iommu_sysclass);
3084
3085         return error;
3086 }
3087
3088 #else
3089 static int __init init_iommu_sysfs(void)
3090 {
3091         return 0;
3092 }
3093 #endif  /* CONFIG_PM */
3094
3095 int __init intel_iommu_init(void)
3096 {
3097         int ret = 0;
3098
3099         if (dmar_table_init())
3100                 return  -ENODEV;
3101
3102         if (dmar_dev_scope_init())
3103                 return  -ENODEV;
3104
3105         /*
3106          * Check the need for DMA-remapping initialization now.
3107          * Above initialization will also be used by Interrupt-remapping.
3108          */
3109         if (no_iommu || (swiotlb && !iommu_pass_through) || dmar_disabled)
3110                 return -ENODEV;
3111
3112         iommu_init_mempool();
3113         dmar_init_reserved_ranges();
3114
3115         init_no_remapping_devices();
3116
3117         ret = init_dmars();
3118         if (ret) {
3119                 printk(KERN_ERR "IOMMU: dmar init failed\n");
3120                 put_iova_domain(&reserved_iova_list);
3121                 iommu_exit_mempool();
3122                 return ret;
3123         }
3124         printk(KERN_INFO
3125         "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3126
3127         init_timer(&unmap_timer);
3128         force_iommu = 1;
3129
3130         if (!iommu_pass_through) {
3131                 printk(KERN_INFO
3132                        "Multi-level page-table translation for DMAR.\n");
3133                 dma_ops = &intel_dma_ops;
3134         } else
3135                 printk(KERN_INFO
3136                        "DMAR: Pass through translation for DMAR.\n");
3137
3138         init_iommu_sysfs();
3139
3140         register_iommu(&intel_iommu_ops);
3141
3142         return 0;
3143 }
3144
3145 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3146                                            struct pci_dev *pdev)
3147 {
3148         struct pci_dev *tmp, *parent;
3149
3150         if (!iommu || !pdev)
3151                 return;
3152
3153         /* dependent device detach */
3154         tmp = pci_find_upstream_pcie_bridge(pdev);
3155         /* Secondary interface's bus number and devfn 0 */
3156         if (tmp) {
3157                 parent = pdev->bus->self;
3158                 while (parent != tmp) {
3159                         iommu_detach_dev(iommu, parent->bus->number,
3160                                          parent->devfn);
3161                         parent = parent->bus->self;
3162                 }
3163                 if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
3164                         iommu_detach_dev(iommu,
3165                                 tmp->subordinate->number, 0);
3166                 else /* this is a legacy PCI bridge */
3167                         iommu_detach_dev(iommu, tmp->bus->number,
3168                                          tmp->devfn);
3169         }
3170 }
3171
3172 static void domain_remove_one_dev_info(struct dmar_domain *domain,
3173                                           struct pci_dev *pdev)
3174 {
3175         struct device_domain_info *info;
3176         struct intel_iommu *iommu;
3177         unsigned long flags;
3178         int found = 0;
3179         struct list_head *entry, *tmp;
3180
3181         iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3182                                 pdev->devfn);
3183         if (!iommu)
3184                 return;
3185
3186         spin_lock_irqsave(&device_domain_lock, flags);
3187         list_for_each_safe(entry, tmp, &domain->devices) {
3188                 info = list_entry(entry, struct device_domain_info, link);
3189                 /* No need to compare PCI domain; it has to be the same */
3190                 if (info->bus == pdev->bus->number &&
3191                     info->devfn == pdev->devfn) {
3192                         list_del(&info->link);
3193                         list_del(&info->global);
3194                         if (info->dev)
3195                                 info->dev->dev.archdata.iommu = NULL;
3196                         spin_unlock_irqrestore(&device_domain_lock, flags);
3197
3198                         iommu_disable_dev_iotlb(info);
3199                         iommu_detach_dev(iommu, info->bus, info->devfn);
3200                         iommu_detach_dependent_devices(iommu, pdev);
3201                         free_devinfo_mem(info);
3202
3203                         spin_lock_irqsave(&device_domain_lock, flags);
3204
3205                         if (found)
3206                                 break;
3207                         else
3208                                 continue;
3209                 }
3210
3211                 /* if there is no other devices under the same iommu
3212                  * owned by this domain, clear this iommu in iommu_bmp
3213                  * update iommu count and coherency
3214                  */
3215                 if (iommu == device_to_iommu(info->segment, info->bus,
3216                                             info->devfn))
3217                         found = 1;
3218         }
3219
3220         if (found == 0) {
3221                 unsigned long tmp_flags;
3222                 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
3223                 clear_bit(iommu->seq_id, &domain->iommu_bmp);
3224                 domain->iommu_count--;
3225                 domain_update_iommu_cap(domain);
3226                 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
3227         }
3228
3229         spin_unlock_irqrestore(&device_domain_lock, flags);
3230 }
3231
3232 static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3233 {
3234         struct device_domain_info *info;
3235         struct intel_iommu *iommu;
3236         unsigned long flags1, flags2;
3237
3238         spin_lock_irqsave(&device_domain_lock, flags1);
3239         while (!list_empty(&domain->devices)) {
3240                 info = list_entry(domain->devices.next,
3241                         struct device_domain_info, link);
3242                 list_del(&info->link);
3243                 list_del(&info->global);
3244                 if (info->dev)
3245                         info->dev->dev.archdata.iommu = NULL;
3246
3247                 spin_unlock_irqrestore(&device_domain_lock, flags1);
3248
3249                 iommu_disable_dev_iotlb(info);
3250                 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
3251                 iommu_detach_dev(iommu, info->bus, info->devfn);
3252                 iommu_detach_dependent_devices(iommu, info->dev);
3253
3254                 /* clear this iommu in iommu_bmp, update iommu count
3255                  * and capabilities
3256                  */
3257                 spin_lock_irqsave(&domain->iommu_lock, flags2);
3258                 if (test_and_clear_bit(iommu->seq_id,
3259                                        &domain->iommu_bmp)) {
3260                         domain->iommu_count--;
3261                         domain_update_iommu_cap(domain);
3262                 }
3263                 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3264
3265                 free_devinfo_mem(info);
3266                 spin_lock_irqsave(&device_domain_lock, flags1);
3267         }
3268         spin_unlock_irqrestore(&device_domain_lock, flags1);
3269 }
3270
3271 /* domain id for virtual machine, it won't be set in context */
3272 static unsigned long vm_domid;
3273
3274 static int vm_domain_min_agaw(struct dmar_domain *domain)
3275 {
3276         int i;
3277         int min_agaw = domain->agaw;
3278
3279         i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
3280         for (; i < g_num_of_iommus; ) {
3281                 if (min_agaw > g_iommus[i]->agaw)
3282                         min_agaw = g_iommus[i]->agaw;
3283
3284                 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
3285         }
3286
3287         return min_agaw;
3288 }
3289
3290 static struct dmar_domain *iommu_alloc_vm_domain(void)
3291 {
3292         struct dmar_domain *domain;
3293
3294         domain = alloc_domain_mem();
3295         if (!domain)
3296                 return NULL;
3297
3298         domain->id = vm_domid++;
3299         memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
3300         domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3301
3302         return domain;
3303 }
3304
3305 static int md_domain_init(struct dmar_domain *domain, int guest_width)
3306 {
3307         int adjust_width;
3308
3309         init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
3310         spin_lock_init(&domain->mapping_lock);
3311         spin_lock_init(&domain->iommu_lock);
3312
3313         domain_reserve_special_ranges(domain);
3314
3315         /* calculate AGAW */
3316         domain->gaw = guest_width;
3317         adjust_width = guestwidth_to_adjustwidth(guest_width);
3318         domain->agaw = width_to_agaw(adjust_width);
3319
3320         INIT_LIST_HEAD(&domain->devices);
3321
3322         domain->iommu_count = 0;
3323         domain->iommu_coherency = 0;
3324         domain->max_addr = 0;
3325
3326         /* always allocate the top pgd */
3327         domain->pgd = (struct dma_pte *)alloc_pgtable_page();
3328         if (!domain->pgd)
3329                 return -ENOMEM;
3330         domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3331         return 0;
3332 }
3333
3334 static void iommu_free_vm_domain(struct dmar_domain *domain)
3335 {
3336         unsigned long flags;
3337         struct dmar_drhd_unit *drhd;
3338         struct intel_iommu *iommu;
3339         unsigned long i;
3340         unsigned long ndomains;
3341
3342         for_each_drhd_unit(drhd) {
3343                 if (drhd->ignored)
3344                         continue;
3345                 iommu = drhd->iommu;
3346
3347                 ndomains = cap_ndoms(iommu->cap);
3348                 i = find_first_bit(iommu->domain_ids, ndomains);
3349                 for (; i < ndomains; ) {
3350                         if (iommu->domains[i] == domain) {
3351                                 spin_lock_irqsave(&iommu->lock, flags);
3352                                 clear_bit(i, iommu->domain_ids);
3353                                 iommu->domains[i] = NULL;
3354                                 spin_unlock_irqrestore(&iommu->lock, flags);
3355                                 break;
3356                         }
3357                         i = find_next_bit(iommu->domain_ids, ndomains, i+1);
3358                 }
3359         }
3360 }
3361
3362 static void vm_domain_exit(struct dmar_domain *domain)
3363 {
3364         /* Domain 0 is reserved, so dont process it */
3365         if (!domain)
3366                 return;
3367
3368         vm_domain_remove_all_dev_info(domain);
3369         /* destroy iovas */
3370         put_iova_domain(&domain->iovad);
3371
3372         /* clear ptes */
3373         dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3374
3375         /* free page tables */
3376         dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3377
3378         iommu_free_vm_domain(domain);
3379         free_domain_mem(domain);
3380 }
3381
3382 static int intel_iommu_domain_init(struct iommu_domain *domain)
3383 {
3384         struct dmar_domain *dmar_domain;
3385
3386         dmar_domain = iommu_alloc_vm_domain();
3387         if (!dmar_domain) {
3388                 printk(KERN_ERR
3389                         "intel_iommu_domain_init: dmar_domain == NULL\n");
3390                 return -ENOMEM;
3391         }
3392         if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
3393                 printk(KERN_ERR
3394                         "intel_iommu_domain_init() failed\n");
3395                 vm_domain_exit(dmar_domain);
3396                 return -ENOMEM;
3397         }
3398         domain->priv = dmar_domain;
3399
3400         return 0;
3401 }
3402
3403 static void intel_iommu_domain_destroy(struct iommu_domain *domain)
3404 {
3405         struct dmar_domain *dmar_domain = domain->priv;
3406
3407         domain->priv = NULL;
3408         vm_domain_exit(dmar_domain);
3409 }
3410
3411 static int intel_iommu_attach_device(struct iommu_domain *domain,
3412                                      struct device *dev)
3413 {
3414         struct dmar_domain *dmar_domain = domain->priv;
3415         struct pci_dev *pdev = to_pci_dev(dev);
3416         struct intel_iommu *iommu;
3417         int addr_width;
3418         u64 end;
3419         int ret;
3420
3421         /* normally pdev is not mapped */
3422         if (unlikely(domain_context_mapped(pdev))) {
3423                 struct dmar_domain *old_domain;
3424
3425                 old_domain = find_domain(pdev);
3426                 if (old_domain) {
3427                         if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
3428                             dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
3429                                 domain_remove_one_dev_info(old_domain, pdev);
3430                         else
3431                                 domain_remove_dev_info(old_domain);
3432                 }
3433         }
3434
3435         iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3436                                 pdev->devfn);
3437         if (!iommu)
3438                 return -ENODEV;
3439
3440         /* check if this iommu agaw is sufficient for max mapped address */
3441         addr_width = agaw_to_width(iommu->agaw);
3442         end = DOMAIN_MAX_ADDR(addr_width);
3443         end = end & VTD_PAGE_MASK;
3444         if (end < dmar_domain->max_addr) {
3445                 printk(KERN_ERR "%s: iommu agaw (%d) is not "
3446                        "sufficient for the mapped address (%llx)\n",
3447                        __func__, iommu->agaw, dmar_domain->max_addr);
3448                 return -EFAULT;
3449         }
3450
3451         ret = domain_add_dev_info(dmar_domain, pdev);
3452         if (ret)
3453                 return ret;
3454
3455         ret = domain_context_mapping(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
3456         return ret;
3457 }
3458
3459 static void intel_iommu_detach_device(struct iommu_domain *domain,
3460                                       struct device *dev)
3461 {
3462         struct dmar_domain *dmar_domain = domain->priv;
3463         struct pci_dev *pdev = to_pci_dev(dev);
3464
3465         domain_remove_one_dev_info(dmar_domain, pdev);
3466 }
3467
3468 static int intel_iommu_map_range(struct iommu_domain *domain,
3469                                  unsigned long iova, phys_addr_t hpa,
3470                                  size_t size, int iommu_prot)
3471 {
3472         struct dmar_domain *dmar_domain = domain->priv;
3473         u64 max_addr;
3474         int addr_width;
3475         int prot = 0;
3476         int ret;
3477
3478         if (iommu_prot & IOMMU_READ)
3479                 prot |= DMA_PTE_READ;
3480         if (iommu_prot & IOMMU_WRITE)
3481                 prot |= DMA_PTE_WRITE;
3482         if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
3483                 prot |= DMA_PTE_SNP;
3484
3485         max_addr = iova + size;
3486         if (dmar_domain->max_addr < max_addr) {
3487                 int min_agaw;
3488                 u64 end;
3489
3490                 /* check if minimum agaw is sufficient for mapped address */
3491                 min_agaw = vm_domain_min_agaw(dmar_domain);
3492                 addr_width = agaw_to_width(min_agaw);
3493                 end = DOMAIN_MAX_ADDR(addr_width);
3494                 end = end & VTD_PAGE_MASK;
3495                 if (end < max_addr) {
3496                         printk(KERN_ERR "%s: iommu agaw (%d) is not "
3497                                "sufficient for the mapped address (%llx)\n",
3498                                __func__, min_agaw, max_addr);
3499                         return -EFAULT;
3500                 }
3501                 dmar_domain->max_addr = max_addr;
3502         }
3503         /* Round up size to next multiple of PAGE_SIZE, if it and
3504            the low bits of hpa would take us onto the next page */
3505         size = aligned_nrpages(hpa, size);
3506         ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
3507                                  hpa >> VTD_PAGE_SHIFT, size, prot);
3508         return ret;
3509 }
3510
3511 static void intel_iommu_unmap_range(struct iommu_domain *domain,
3512                                     unsigned long iova, size_t size)
3513 {
3514         struct dmar_domain *dmar_domain = domain->priv;
3515
3516         dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
3517                             (iova + size - 1) >> VTD_PAGE_SHIFT);
3518
3519         if (dmar_domain->max_addr == iova + size)
3520                 dmar_domain->max_addr = iova;
3521 }
3522
3523 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
3524                                             unsigned long iova)
3525 {
3526         struct dmar_domain *dmar_domain = domain->priv;
3527         struct dma_pte *pte;
3528         u64 phys = 0;
3529
3530         pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT);
3531         if (pte)
3532                 phys = dma_pte_addr(pte);
3533
3534         return phys;
3535 }
3536
3537 static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
3538                                       unsigned long cap)
3539 {
3540         struct dmar_domain *dmar_domain = domain->priv;
3541
3542         if (cap == IOMMU_CAP_CACHE_COHERENCY)
3543                 return dmar_domain->iommu_snooping;
3544
3545         return 0;
3546 }
3547
3548 static struct iommu_ops intel_iommu_ops = {
3549         .domain_init    = intel_iommu_domain_init,
3550         .domain_destroy = intel_iommu_domain_destroy,
3551         .attach_dev     = intel_iommu_attach_device,
3552         .detach_dev     = intel_iommu_detach_device,
3553         .map            = intel_iommu_map_range,
3554         .unmap          = intel_iommu_unmap_range,
3555         .iova_to_phys   = intel_iommu_iova_to_phys,
3556         .domain_has_cap = intel_iommu_domain_has_cap,
3557 };
3558
3559 static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
3560 {
3561         /*
3562          * Mobile 4 Series Chipset neglects to set RWBF capability,
3563          * but needs it:
3564          */
3565         printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
3566         rwbf_quirk = 1;
3567 }
3568
3569 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);