drivers/pci/dmar.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  *
  22  * This file implements early detection/parsing of Remapping Devices
  23  * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  24  * tables.
  25  *
  26  * These routines are used by both DMA-remapping and Interrupt-remapping
  27  */
  28
  29 #include <linux/pci.h>
  30 #include <linux/dmar.h>
  31 #include <linux/iova.h>
  32 #include <linux/intel-iommu.h>
  33 #include <linux/timer.h>
  34 #include <linux/irq.h>
  35 #include <linux/interrupt.h>
  36 #include <linux/dmi.h>
  37
  38 #undef PREFIX
  39 #define PREFIX "DMAR:"
  40
  41 /* No locks are needed as DMA remapping hardware unit
  42  * list is constructed at boot time and hotplug of
  43  * these units are not supported by the architecture.
  44  */
  45 LIST_HEAD(dmar_drhd_units);
  46
  47 static struct acpi_table_header * __initdata dmar_tbl;
  48 static acpi_size dmar_tbl_size;
  49
  50 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  51 {
  52         /*
  53          * add INCLUDE_ALL at the tail, so scan the list will find it at
  54          * the very end.
  55          */
  56         if (drhd->include_all)
  57                 list_add_tail(&drhd->list, &dmar_drhd_units);
  58         else
  59                 list_add(&drhd->list, &dmar_drhd_units);
  60 }
  61
  62 static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
  63                                            struct pci_dev **dev, u16 segment)
  64 {
  65         struct pci_bus *bus;
  66         struct pci_dev *pdev = NULL;
  67         struct acpi_dmar_pci_path *path;
  68         int count;
  69
  70         bus = pci_find_bus(segment, scope->bus);
  71         path = (struct acpi_dmar_pci_path *)(scope + 1);
  72         count = (scope->length - sizeof(struct acpi_dmar_device_scope))
  73                 / sizeof(struct acpi_dmar_pci_path);
  74
  75         while (count) {
  76                 if (pdev)
  77                         pci_dev_put(pdev);
  78                 /*
  79                  * Some BIOSes list non-exist devices in DMAR table, just
  80                  * ignore it
  81                  */
  82                 if (!bus) {
  83                         printk(KERN_WARNING
  84                         PREFIX "Device scope bus [%d] not found\n",
  85                         scope->bus);
  86                         break;
  87                 }
  88                 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
  89                 if (!pdev) {
  90                         printk(KERN_WARNING PREFIX
  91                         "Device scope device [%04x:%02x:%02x.%02x] not found\n",
  92                                 segment, bus->number, path->dev, path->fn);
  93                         break;
  94                 }
  95                 path ++;
  96                 count --;
  97                 bus = pdev->subordinate;
  98         }
  99         if (!pdev) {
 100                 printk(KERN_WARNING PREFIX
 101                 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
 102                 segment, scope->bus, path->dev, path->fn);
 103                 *dev = NULL;
 104                 return 0;
 105         }
 106         if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
 107                         pdev->subordinate) || (scope->entry_type == \
 108                         ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
 109                 pci_dev_put(pdev);
 110                 printk(KERN_WARNING PREFIX
 111                         "Device scope type does not match for %s\n",
 112                          pci_name(pdev));
 113                 return -EINVAL;
 114         }
 115         *dev = pdev;
 116         return 0;
 117 }
 118
 119 static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
 120                                        struct pci_dev ***devices, u16 segment)
 121 {
 122         struct acpi_dmar_device_scope *scope;
 123         void * tmp = start;
 124         int index;
 125         int ret;
 126
 127         *cnt = 0;
 128         while (start < end) {
 129                 scope = start;
 130                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 131                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 132                         (*cnt)++;
 133                 else
 134                         printk(KERN_WARNING PREFIX
 135                                 "Unsupported device scope\n");
 136                 start += scope->length;
 137         }
 138         if (*cnt == 0)
 139                 return 0;
 140
 141         *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
 142         if (!*devices)
 143                 return -ENOMEM;
 144
 145         start = tmp;
 146         index = 0;
 147         while (start < end) {
 148                 scope = start;
 149                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 150                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
 151                         ret = dmar_parse_one_dev_scope(scope,
 152                                 &(*devices)[index], segment);
 153                         if (ret) {
 154                                 kfree(*devices);
 155                                 return ret;
 156                         }
 157                         index ++;
 158                 }
 159                 start += scope->length;
 160         }
 161
 162         return 0;
 163 }
 164
 165 /**
 166  * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 167  * structure which uniquely represent one DMA remapping hardware unit
 168  * present in the platform
 169  */
 170 static int __init
 171 dmar_parse_one_drhd(struct acpi_dmar_header *header)
 172 {
 173         struct acpi_dmar_hardware_unit *drhd;
 174         struct dmar_drhd_unit *dmaru;
 175         int ret = 0;
 176
 177         drhd = (struct acpi_dmar_hardware_unit *)header;
 178         if (!drhd->address) {
 179                 /* Promote an attitude of violence to a BIOS engineer today */
 180                 WARN(1, "Your BIOS is broken; DMAR reported at address zero!\n"
 181                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 182                      dmi_get_system_info(DMI_BIOS_VENDOR),
 183                      dmi_get_system_info(DMI_BIOS_VERSION),
 184                      dmi_get_system_info(DMI_PRODUCT_VERSION));
 185                 return -ENODEV;
 186         }
 187         dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
 188         if (!dmaru)
 189                 return -ENOMEM;
 190
 191         dmaru->hdr = header;
 192         dmaru->reg_base_addr = drhd->address;
 193         dmaru->segment = drhd->segment;
 194         dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
 195
 196         ret = alloc_iommu(dmaru);
 197         if (ret) {
 198                 kfree(dmaru);
 199                 return ret;
 200         }
 201         dmar_register_drhd_unit(dmaru);
 202         return 0;
 203 }
 204
 205 static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
 206 {
 207         struct acpi_dmar_hardware_unit *drhd;
 208         int ret = 0;
 209
 210         drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
 211
 212         if (dmaru->include_all)
 213                 return 0;
 214
 215         ret = dmar_parse_dev_scope((void *)(drhd + 1),
 216                                 ((void *)drhd) + drhd->header.length,
 217                                 &dmaru->devices_cnt, &dmaru->devices,
 218                                 drhd->segment);
 219         if (ret) {
 220                 list_del(&dmaru->list);
 221                 kfree(dmaru);
 222         }
 223         return ret;
 224 }
 225
 226 #ifdef CONFIG_DMAR
 227 LIST_HEAD(dmar_rmrr_units);
 228
 229 static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
 230 {
 231         list_add(&rmrr->list, &dmar_rmrr_units);
 232 }
 233
 234
 235 static int __init
 236 dmar_parse_one_rmrr(struct acpi_dmar_header *header)
 237 {
 238         struct acpi_dmar_reserved_memory *rmrr;
 239         struct dmar_rmrr_unit *rmrru;
 240
 241         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
 242         if (!rmrru)
 243                 return -ENOMEM;
 244
 245         rmrru->hdr = header;
 246         rmrr = (struct acpi_dmar_reserved_memory *)header;
 247         rmrru->base_address = rmrr->base_address;
 248         rmrru->end_address = rmrr->end_address;
 249
 250         dmar_register_rmrr_unit(rmrru);
 251         return 0;
 252 }
 253
 254 static int __init
 255 rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
 256 {
 257         struct acpi_dmar_reserved_memory *rmrr;
 258         int ret;
 259
 260         rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
 261         ret = dmar_parse_dev_scope((void *)(rmrr + 1),
 262                 ((void *)rmrr) + rmrr->header.length,
 263                 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
 264
 265         if (ret || (rmrru->devices_cnt == 0)) {
 266                 list_del(&rmrru->list);
 267                 kfree(rmrru);
 268         }
 269         return ret;
 270 }
 271
 272 static LIST_HEAD(dmar_atsr_units);
 273
 274 static int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
 275 {
 276         struct acpi_dmar_atsr *atsr;
 277         struct dmar_atsr_unit *atsru;
 278
 279         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
 280         atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
 281         if (!atsru)
 282                 return -ENOMEM;
 283
 284         atsru->hdr = hdr;
 285         atsru->include_all = atsr->flags & 0x1;
 286
 287         list_add(&atsru->list, &dmar_atsr_units);
 288
 289         return 0;
 290 }
 291
 292 static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
 293 {
 294         int rc;
 295         struct acpi_dmar_atsr *atsr;
 296
 297         if (atsru->include_all)
 298                 return 0;
 299
 300         atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
 301         rc = dmar_parse_dev_scope((void *)(atsr + 1),
 302                                 (void *)atsr + atsr->header.length,
 303                                 &atsru->devices_cnt, &atsru->devices,
 304                                 atsr->segment);
 305         if (rc || !atsru->devices_cnt) {
 306                 list_del(&atsru->list);
 307                 kfree(atsru);
 308         }
 309
 310         return rc;
 311 }
 312
 313 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
 314 {
 315         int i;
 316         struct pci_bus *bus;
 317         struct acpi_dmar_atsr *atsr;
 318         struct dmar_atsr_unit *atsru;
 319
 320         list_for_each_entry(atsru, &dmar_atsr_units, list) {
 321                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
 322                 if (atsr->segment == pci_domain_nr(dev->bus))
 323                         goto found;
 324         }
 325
 326         return 0;
 327
 328 found:
 329         for (bus = dev->bus; bus; bus = bus->parent) {
 330                 struct pci_dev *bridge = bus->self;
 331
 332                 if (!bridge || !bridge->is_pcie ||
 333                     bridge->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
 334                         return 0;
 335
 336                 if (bridge->pcie_type == PCI_EXP_TYPE_ROOT_PORT) {
 337                         for (i = 0; i < atsru->devices_cnt; i++)
 338                                 if (atsru->devices[i] == bridge)
 339                                         return 1;
 340                         break;
 341                 }
 342         }
 343
 344         if (atsru->include_all)
 345                 return 1;
 346
 347         return 0;
 348 }
 349 #endif
 350
 351 static void __init
 352 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
 353 {
 354         struct acpi_dmar_hardware_unit *drhd;
 355         struct acpi_dmar_reserved_memory *rmrr;
 356         struct acpi_dmar_atsr *atsr;
 357
 358         switch (header->type) {
 359         case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 360                 drhd = container_of(header, struct acpi_dmar_hardware_unit,
 361                                     header);
 362                 printk (KERN_INFO PREFIX
 363                         "DRHD base: %#016Lx flags: %#x\n",
 364                         (unsigned long long)drhd->address, drhd->flags);
 365                 break;
 366         case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 367                 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
 368                                     header);
 369                 printk (KERN_INFO PREFIX
 370                         "RMRR base: %#016Lx end: %#016Lx\n",
 371                         (unsigned long long)rmrr->base_address,
 372                         (unsigned long long)rmrr->end_address);
 373                 break;
 374         case ACPI_DMAR_TYPE_ATSR:
 375                 atsr = container_of(header, struct acpi_dmar_atsr, header);
 376                 printk(KERN_INFO PREFIX "ATSR flags: %#x\n", atsr->flags);
 377                 break;
 378         }
 379 }
 380
 381 /**
 382  * dmar_table_detect - checks to see if the platform supports DMAR devices
 383  */
 384 static int __init dmar_table_detect(void)
 385 {
 386         acpi_status status = AE_OK;
 387
 388         /* if we could find DMAR table, then there are DMAR devices */
 389         status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
 390                                 (struct acpi_table_header **)&dmar_tbl,
 391                                 &dmar_tbl_size);
 392
 393         if (ACPI_SUCCESS(status) && !dmar_tbl) {
 394                 printk (KERN_WARNING PREFIX "Unable to map DMAR\n");
 395                 status = AE_NOT_FOUND;
 396         }
 397
 398         return (ACPI_SUCCESS(status) ? 1 : 0);
 399 }
 400
 401 /**
 402  * parse_dmar_table - parses the DMA reporting table
 403  */
 404 static int __init
 405 parse_dmar_table(void)
 406 {
 407         struct acpi_table_dmar *dmar;
 408         struct acpi_dmar_header *entry_header;
 409         int ret = 0;
 410
 411         /*
 412          * Do it again, earlier dmar_tbl mapping could be mapped with
 413          * fixed map.
 414          */
 415         dmar_table_detect();
 416
 417         dmar = (struct acpi_table_dmar *)dmar_tbl;
 418         if (!dmar)
 419                 return -ENODEV;
 420
 421         if (dmar->width < PAGE_SHIFT - 1) {
 422                 printk(KERN_WARNING PREFIX "Invalid DMAR haw\n");
 423                 return -EINVAL;
 424         }
 425
 426         printk (KERN_INFO PREFIX "Host address width %d\n",
 427                 dmar->width + 1);
 428
 429         entry_header = (struct acpi_dmar_header *)(dmar + 1);
 430         while (((unsigned long)entry_header) <
 431                         (((unsigned long)dmar) + dmar_tbl->length)) {
 432                 /* Avoid looping forever on bad ACPI tables */
 433                 if (entry_header->length == 0) {
 434                         printk(KERN_WARNING PREFIX
 435                                 "Invalid 0-length structure\n");
 436                         ret = -EINVAL;
 437                         break;
 438                 }
 439
 440                 dmar_table_print_dmar_entry(entry_header);
 441
 442                 switch (entry_header->type) {
 443                 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 444                         ret = dmar_parse_one_drhd(entry_header);
 445                         break;
 446                 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 447 #ifdef CONFIG_DMAR
 448                         ret = dmar_parse_one_rmrr(entry_header);
 449 #endif
 450                         break;
 451                 case ACPI_DMAR_TYPE_ATSR:
 452 #ifdef CONFIG_DMAR
 453                         ret = dmar_parse_one_atsr(entry_header);
 454 #endif
 455                         break;
 456                 default:
 457                         printk(KERN_WARNING PREFIX
 458                                 "Unknown DMAR structure type\n");
 459                         ret = 0; /* for forward compatibility */
 460                         break;
 461                 }
 462                 if (ret)
 463                         break;
 464
 465                 entry_header = ((void *)entry_header + entry_header->length);
 466         }
 467         return ret;
 468 }
 469
 470 int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
 471                           struct pci_dev *dev)
 472 {
 473         int index;
 474
 475         while (dev) {
 476                 for (index = 0; index < cnt; index++)
 477                         if (dev == devices[index])
 478                                 return 1;
 479
 480                 /* Check our parent */
 481                 dev = dev->bus->self;
 482         }
 483
 484         return 0;
 485 }
 486
 487 struct dmar_drhd_unit *
 488 dmar_find_matched_drhd_unit(struct pci_dev *dev)
 489 {
 490         struct dmar_drhd_unit *dmaru = NULL;
 491         struct acpi_dmar_hardware_unit *drhd;
 492
 493         list_for_each_entry(dmaru, &dmar_drhd_units, list) {
 494                 drhd = container_of(dmaru->hdr,
 495                                     struct acpi_dmar_hardware_unit,
 496                                     header);
 497
 498                 if (dmaru->include_all &&
 499                     drhd->segment == pci_domain_nr(dev->bus))
 500                         return dmaru;
 501
 502                 if (dmar_pci_device_match(dmaru->devices,
 503                                           dmaru->devices_cnt, dev))
 504                         return dmaru;
 505         }
 506
 507         return NULL;
 508 }
 509
 510 int __init dmar_dev_scope_init(void)
 511 {
 512         struct dmar_drhd_unit *drhd, *drhd_n;
 513         int ret = -ENODEV;
 514
 515         list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
 516                 ret = dmar_parse_dev(drhd);
 517                 if (ret)
 518                         return ret;
 519         }
 520
 521 #ifdef CONFIG_DMAR
 522         {
 523                 struct dmar_rmrr_unit *rmrr, *rmrr_n;
 524                 struct dmar_atsr_unit *atsr, *atsr_n;
 525
 526                 list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
 527                         ret = rmrr_parse_dev(rmrr);
 528                         if (ret)
 529                                 return ret;
 530                 }
 531
 532                 list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
 533                         ret = atsr_parse_dev(atsr);
 534                         if (ret)
 535                                 return ret;
 536                 }
 537         }
 538 #endif
 539
 540         return ret;
 541 }
 542
 543
 544 int __init dmar_table_init(void)
 545 {
 546         static int dmar_table_initialized;
 547         int ret;
 548
 549         if (dmar_table_initialized)
 550                 return 0;
 551
 552         dmar_table_initialized = 1;
 553
 554         ret = parse_dmar_table();
 555         if (ret) {
 556                 if (ret != -ENODEV)
 557                         printk(KERN_INFO PREFIX "parse DMAR table failure.\n");
 558                 return ret;
 559         }
 560
 561         if (list_empty(&dmar_drhd_units)) {
 562                 printk(KERN_INFO PREFIX "No DMAR devices found\n");
 563                 return -ENODEV;
 564         }
 565
 566 #ifdef CONFIG_DMAR
 567         if (list_empty(&dmar_rmrr_units))
 568                 printk(KERN_INFO PREFIX "No RMRR found\n");
 569
 570         if (list_empty(&dmar_atsr_units))
 571                 printk(KERN_INFO PREFIX "No ATSR found\n");
 572 #endif
 573
 574 #ifdef CONFIG_INTR_REMAP
 575         parse_ioapics_under_ir();
 576 #endif
 577         return 0;
 578 }
 579
 580 void __init detect_intel_iommu(void)
 581 {
 582         int ret;
 583
 584         ret = dmar_table_detect();
 585
 586         {
 587 #ifdef CONFIG_INTR_REMAP
 588                 struct acpi_table_dmar *dmar;
 589                 /*
 590                  * for now we will disable dma-remapping when interrupt
 591                  * remapping is enabled.
 592                  * When support for queued invalidation for IOTLB invalidation
 593                  * is added, we will not need this any more.
 594                  */
 595                 dmar = (struct acpi_table_dmar *) dmar_tbl;
 596                 if (ret && cpu_has_x2apic && dmar->flags & 0x1)
 597                         printk(KERN_INFO
 598                                "Queued invalidation will be enabled to support "
 599                                "x2apic and Intr-remapping.\n");
 600 #endif
 601 #ifdef CONFIG_DMAR
 602                 if (ret && !no_iommu && !iommu_detected && !swiotlb &&
 603                     !dmar_disabled)
 604                         iommu_detected = 1;
 605 #endif
 606         }
 607         early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
 608         dmar_tbl = NULL;
 609 }
 610
 611
 612 int alloc_iommu(struct dmar_drhd_unit *drhd)
 613 {
 614         struct intel_iommu *iommu;
 615         int map_size;
 616         u32 ver;
 617         static int iommu_allocated = 0;
 618         int agaw = 0;
 619         int msagaw = 0;
 620
 621         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
 622         if (!iommu)
 623                 return -ENOMEM;
 624
 625         iommu->seq_id = iommu_allocated++;
 626         sprintf (iommu->name, "dmar%d", iommu->seq_id);
 627
 628         iommu->reg = ioremap(drhd->reg_base_addr, VTD_PAGE_SIZE);
 629         if (!iommu->reg) {
 630                 printk(KERN_ERR "IOMMU: can't map the region\n");
 631                 goto error;
 632         }
 633         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 634         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 635
 636 #ifdef CONFIG_DMAR
 637         agaw = iommu_calculate_agaw(iommu);
 638         if (agaw < 0) {
 639                 printk(KERN_ERR
 640                        "Cannot get a valid agaw for iommu (seq_id = %d)\n",
 641                        iommu->seq_id);
 642                 goto error;
 643         }
 644         msagaw = iommu_calculate_max_sagaw(iommu);
 645         if (msagaw < 0) {
 646                 printk(KERN_ERR
 647                         "Cannot get a valid max agaw for iommu (seq_id = %d)\n",
 648                         iommu->seq_id);
 649                 goto error;
 650         }
 651 #endif
 652         iommu->agaw = agaw;
 653         iommu->msagaw = msagaw;
 654
 655         /* the registers might be more than one page */
 656         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 657                 cap_max_fault_reg_offset(iommu->cap));
 658         map_size = VTD_PAGE_ALIGN(map_size);
 659         if (map_size > VTD_PAGE_SIZE) {
 660                 iounmap(iommu->reg);
 661                 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
 662                 if (!iommu->reg) {
 663                         printk(KERN_ERR "IOMMU: can't map the region\n");
 664                         goto error;
 665                 }
 666         }
 667
 668         ver = readl(iommu->reg + DMAR_VER_REG);
 669         pr_debug("IOMMU %llx: ver %d:%d cap %llx ecap %llx\n",
 670                 (unsigned long long)drhd->reg_base_addr,
 671                 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
 672                 (unsigned long long)iommu->cap,
 673                 (unsigned long long)iommu->ecap);
 674
 675         spin_lock_init(&iommu->register_lock);
 676
 677         drhd->iommu = iommu;
 678         return 0;
 679 error:
 680         kfree(iommu);
 681         return -1;
 682 }
 683
 684 void free_iommu(struct intel_iommu *iommu)
 685 {
 686         if (!iommu)
 687                 return;
 688
 689 #ifdef CONFIG_DMAR
 690         free_dmar_iommu(iommu);
 691 #endif
 692
 693         if (iommu->reg)
 694                 iounmap(iommu->reg);
 695         kfree(iommu);
 696 }
 697
 698 /*
 699  * Reclaim all the submitted descriptors which have completed its work.
 700  */
 701 static inline void reclaim_free_desc(struct q_inval *qi)
 702 {
 703         while (qi->desc_status[qi->free_tail] == QI_DONE ||
 704                qi->desc_status[qi->free_tail] == QI_ABORT) {
 705                 qi->desc_status[qi->free_tail] = QI_FREE;
 706                 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
 707                 qi->free_cnt++;
 708         }
 709 }
 710
 711 static int qi_check_fault(struct intel_iommu *iommu, int index)
 712 {
 713         u32 fault;
 714         int head, tail;
 715         struct q_inval *qi = iommu->qi;
 716         int wait_index = (index + 1) % QI_LENGTH;
 717
 718         if (qi->desc_status[wait_index] == QI_ABORT)
 719                 return -EAGAIN;
 720
 721         fault = readl(iommu->reg + DMAR_FSTS_REG);
 722
 723         /*
 724          * If IQE happens, the head points to the descriptor associated
 725          * with the error. No new descriptors are fetched until the IQE
 726          * is cleared.
 727          */
 728         if (fault & DMA_FSTS_IQE) {
 729                 head = readl(iommu->reg + DMAR_IQH_REG);
 730                 if ((head >> DMAR_IQ_SHIFT) == index) {
 731                         printk(KERN_ERR "VT-d detected invalid descriptor: "
 732                                 "low=%llx, high=%llx\n",
 733                                 (unsigned long long)qi->desc[index].low,
 734                                 (unsigned long long)qi->desc[index].high);
 735                         memcpy(&qi->desc[index], &qi->desc[wait_index],
 736                                         sizeof(struct qi_desc));
 737                         __iommu_flush_cache(iommu, &qi->desc[index],
 738                                         sizeof(struct qi_desc));
 739                         writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
 740                         return -EINVAL;
 741                 }
 742         }
 743
 744         /*
 745          * If ITE happens, all pending wait_desc commands are aborted.
 746          * No new descriptors are fetched until the ITE is cleared.
 747          */
 748         if (fault & DMA_FSTS_ITE) {
 749                 head = readl(iommu->reg + DMAR_IQH_REG);
 750                 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
 751                 head |= 1;
 752                 tail = readl(iommu->reg + DMAR_IQT_REG);
 753                 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
 754
 755                 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
 756
 757                 do {
 758                         if (qi->desc_status[head] == QI_IN_USE)
 759                                 qi->desc_status[head] = QI_ABORT;
 760                         head = (head - 2 + QI_LENGTH) % QI_LENGTH;
 761                 } while (head != tail);
 762
 763                 if (qi->desc_status[wait_index] == QI_ABORT)
 764                         return -EAGAIN;
 765         }
 766
 767         if (fault & DMA_FSTS_ICE)
 768                 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
 769
 770         return 0;
 771 }
 772
 773 /*
 774  * Submit the queued invalidation descriptor to the remapping
 775  * hardware unit and wait for its completion.
 776  */
 777 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
 778 {
 779         int rc;
 780         struct q_inval *qi = iommu->qi;
 781         struct qi_desc *hw, wait_desc;
 782         int wait_index, index;
 783         unsigned long flags;
 784
 785         if (!qi)
 786                 return 0;
 787
 788         hw = qi->desc;
 789
 790 restart:
 791         rc = 0;
 792
 793         spin_lock_irqsave(&qi->q_lock, flags);
 794         while (qi->free_cnt < 3) {
 795                 spin_unlock_irqrestore(&qi->q_lock, flags);
 796                 cpu_relax();
 797                 spin_lock_irqsave(&qi->q_lock, flags);
 798         }
 799
 800         index = qi->free_head;
 801         wait_index = (index + 1) % QI_LENGTH;
 802
 803         qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
 804
 805         hw[index] = *desc;
 806
 807         wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
 808                         QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
 809         wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
 810
 811         hw[wait_index] = wait_desc;
 812
 813         __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
 814         __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
 815
 816         qi->free_head = (qi->free_head + 2) % QI_LENGTH;
 817         qi->free_cnt -= 2;
 818
 819         /*
 820          * update the HW tail register indicating the presence of
 821          * new descriptors.
 822          */
 823         writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
 824
 825         while (qi->desc_status[wait_index] != QI_DONE) {
 826                 /*
 827                  * We will leave the interrupts disabled, to prevent interrupt
 828                  * context to queue another cmd while a cmd is already submitted
 829                  * and waiting for completion on this cpu. This is to avoid
 830                  * a deadlock where the interrupt context can wait indefinitely
 831                  * for free slots in the queue.
 832                  */
 833                 rc = qi_check_fault(iommu, index);
 834                 if (rc)
 835                         break;
 836
 837                 spin_unlock(&qi->q_lock);
 838                 cpu_relax();
 839                 spin_lock(&qi->q_lock);
 840         }
 841
 842         qi->desc_status[index] = QI_DONE;
 843
 844         reclaim_free_desc(qi);
 845         spin_unlock_irqrestore(&qi->q_lock, flags);
 846
 847         if (rc == -EAGAIN)
 848                 goto restart;
 849
 850         return rc;
 851 }
 852
 853 /*
 854  * Flush the global interrupt entry cache.
 855  */
 856 void qi_global_iec(struct intel_iommu *iommu)
 857 {
 858         struct qi_desc desc;
 859
 860         desc.low = QI_IEC_TYPE;
 861         desc.high = 0;
 862
 863         /* should never fail */
 864         qi_submit_sync(&desc, iommu);
 865 }
 866
 867 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
 868                       u64 type)
 869 {
 870         struct qi_desc desc;
 871
 872         desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
 873                         | QI_CC_GRAN(type) | QI_CC_TYPE;
 874         desc.high = 0;
 875
 876         qi_submit_sync(&desc, iommu);
 877 }
 878
 879 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
 880                     unsigned int size_order, u64 type)
 881 {
 882         u8 dw = 0, dr = 0;
 883
 884         struct qi_desc desc;
 885         int ih = 0;
 886
 887         if (cap_write_drain(iommu->cap))
 888                 dw = 1;
 889
 890         if (cap_read_drain(iommu->cap))
 891                 dr = 1;
 892
 893         desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
 894                 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
 895         desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
 896                 | QI_IOTLB_AM(size_order);
 897
 898         qi_submit_sync(&desc, iommu);
 899 }
 900
 901 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
 902                         u64 addr, unsigned mask)
 903 {
 904         struct qi_desc desc;
 905
 906         if (mask) {
 907                 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
 908                 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
 909                 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
 910         } else
 911                 desc.high = QI_DEV_IOTLB_ADDR(addr);
 912
 913         if (qdep >= QI_DEV_IOTLB_MAX_INVS)
 914                 qdep = 0;
 915
 916         desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
 917                    QI_DIOTLB_TYPE;
 918
 919         qi_submit_sync(&desc, iommu);
 920 }
 921
 922 /*
 923  * Disable Queued Invalidation interface.
 924  */
 925 void dmar_disable_qi(struct intel_iommu *iommu)
 926 {
 927         unsigned long flags;
 928         u32 sts;
 929         cycles_t start_time = get_cycles();
 930
 931         if (!ecap_qis(iommu->ecap))
 932                 return;
 933
 934         spin_lock_irqsave(&iommu->register_lock, flags);
 935
 936         sts =  dmar_readq(iommu->reg + DMAR_GSTS_REG);
 937         if (!(sts & DMA_GSTS_QIES))
 938                 goto end;
 939
 940         /*
 941          * Give a chance to HW to complete the pending invalidation requests.
 942          */
 943         while ((readl(iommu->reg + DMAR_IQT_REG) !=
 944                 readl(iommu->reg + DMAR_IQH_REG)) &&
 945                 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
 946                 cpu_relax();
 947
 948         iommu->gcmd &= ~DMA_GCMD_QIE;
 949         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
 950
 951         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
 952                       !(sts & DMA_GSTS_QIES), sts);
 953 end:
 954         spin_unlock_irqrestore(&iommu->register_lock, flags);
 955 }
 956
 957 /*
 958  * Enable queued invalidation.
 959  */
 960 static void __dmar_enable_qi(struct intel_iommu *iommu)
 961 {
 962         u32 sts;
 963         unsigned long flags;
 964         struct q_inval *qi = iommu->qi;
 965
 966         qi->free_head = qi->free_tail = 0;
 967         qi->free_cnt = QI_LENGTH;
 968
 969         spin_lock_irqsave(&iommu->register_lock, flags);
 970
 971         /* write zero to the tail reg */
 972         writel(0, iommu->reg + DMAR_IQT_REG);
 973
 974         dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
 975
 976         iommu->gcmd |= DMA_GCMD_QIE;
 977         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
 978
 979         /* Make sure hardware complete it */
 980         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
 981
 982         spin_unlock_irqrestore(&iommu->register_lock, flags);
 983 }
 984
 985 /*
 986  * Enable Queued Invalidation interface. This is a must to support
 987  * interrupt-remapping. Also used by DMA-remapping, which replaces
 988  * register based IOTLB invalidation.
 989  */
 990 int dmar_enable_qi(struct intel_iommu *iommu)
 991 {
 992         struct q_inval *qi;
 993
 994         if (!ecap_qis(iommu->ecap))
 995                 return -ENOENT;
 996
 997         /*
 998          * queued invalidation is already setup and enabled.
 999          */
1000         if (iommu->qi)
1001                 return 0;
1002
1003         iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1004         if (!iommu->qi)
1005                 return -ENOMEM;
1006
1007         qi = iommu->qi;
1008
1009         qi->desc = (void *)(get_zeroed_page(GFP_ATOMIC));
1010         if (!qi->desc) {
1011                 kfree(qi);
1012                 iommu->qi = 0;
1013                 return -ENOMEM;
1014         }
1015
1016         qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1017         if (!qi->desc_status) {
1018                 free_page((unsigned long) qi->desc);
1019                 kfree(qi);
1020                 iommu->qi = 0;
1021                 return -ENOMEM;
1022         }
1023
1024         qi->free_head = qi->free_tail = 0;
1025         qi->free_cnt = QI_LENGTH;
1026
1027         spin_lock_init(&qi->q_lock);
1028
1029         __dmar_enable_qi(iommu);
1030
1031         return 0;
1032 }
1033
1034 /* iommu interrupt handling. Most stuff are MSI-like. */
1035
1036 enum faulttype {
1037         DMA_REMAP,
1038         INTR_REMAP,
1039         UNKNOWN,
1040 };
1041
1042 static const char *dma_remap_fault_reasons[] =
1043 {
1044         "Software",
1045         "Present bit in root entry is clear",
1046         "Present bit in context entry is clear",
1047         "Invalid context entry",
1048         "Access beyond MGAW",
1049         "PTE Write access is not set",
1050         "PTE Read access is not set",
1051         "Next page table ptr is invalid",
1052         "Root table address invalid",
1053         "Context table ptr is invalid",
1054         "non-zero reserved fields in RTP",
1055         "non-zero reserved fields in CTP",
1056         "non-zero reserved fields in PTE",
1057 };
1058
1059 static const char *intr_remap_fault_reasons[] =
1060 {
1061         "Detected reserved fields in the decoded interrupt-remapped request",
1062         "Interrupt index exceeded the interrupt-remapping table size",
1063         "Present field in the IRTE entry is clear",
1064         "Error accessing interrupt-remapping table pointed by IRTA_REG",
1065         "Detected reserved fields in the IRTE entry",
1066         "Blocked a compatibility format interrupt request",
1067         "Blocked an interrupt request due to source-id verification failure",
1068 };
1069
1070 #define MAX_FAULT_REASON_IDX    (ARRAY_SIZE(fault_reason_strings) - 1)
1071
1072 const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1073 {
1074         if (fault_reason >= 0x20 && (fault_reason <= 0x20 +
1075                                      ARRAY_SIZE(intr_remap_fault_reasons))) {
1076                 *fault_type = INTR_REMAP;
1077                 return intr_remap_fault_reasons[fault_reason - 0x20];
1078         } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1079                 *fault_type = DMA_REMAP;
1080                 return dma_remap_fault_reasons[fault_reason];
1081         } else {
1082                 *fault_type = UNKNOWN;
1083                 return "Unknown";
1084         }
1085 }
1086
1087 void dmar_msi_unmask(unsigned int irq)
1088 {
1089         struct intel_iommu *iommu = get_irq_data(irq);
1090         unsigned long flag;
1091
1092         /* unmask it */
1093         spin_lock_irqsave(&iommu->register_lock, flag);
1094         writel(0, iommu->reg + DMAR_FECTL_REG);
1095         /* Read a reg to force flush the post write */
1096         readl(iommu->reg + DMAR_FECTL_REG);
1097         spin_unlock_irqrestore(&iommu->register_lock, flag);
1098 }
1099
1100 void dmar_msi_mask(unsigned int irq)
1101 {
1102         unsigned long flag;
1103         struct intel_iommu *iommu = get_irq_data(irq);
1104
1105         /* mask it */
1106         spin_lock_irqsave(&iommu->register_lock, flag);
1107         writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1108         /* Read a reg to force flush the post write */
1109         readl(iommu->reg + DMAR_FECTL_REG);
1110         spin_unlock_irqrestore(&iommu->register_lock, flag);
1111 }
1112
1113 void dmar_msi_write(int irq, struct msi_msg *msg)
1114 {
1115         struct intel_iommu *iommu = get_irq_data(irq);
1116         unsigned long flag;
1117
1118         spin_lock_irqsave(&iommu->register_lock, flag);
1119         writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1120         writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1121         writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1122         spin_unlock_irqrestore(&iommu->register_lock, flag);
1123 }
1124
1125 void dmar_msi_read(int irq, struct msi_msg *msg)
1126 {
1127         struct intel_iommu *iommu = get_irq_data(irq);
1128         unsigned long flag;
1129
1130         spin_lock_irqsave(&iommu->register_lock, flag);
1131         msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1132         msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1133         msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1134         spin_unlock_irqrestore(&iommu->register_lock, flag);
1135 }
1136
1137 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1138                 u8 fault_reason, u16 source_id, unsigned long long addr)
1139 {
1140         const char *reason;
1141         int fault_type;
1142
1143         reason = dmar_get_fault_reason(fault_reason, &fault_type);
1144
1145         if (fault_type == INTR_REMAP)
1146                 printk(KERN_ERR "INTR-REMAP: Request device [[%02x:%02x.%d] "
1147                        "fault index %llx\n"
1148                         "INTR-REMAP:[fault reason %02d] %s\n",
1149                         (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1150                         PCI_FUNC(source_id & 0xFF), addr >> 48,
1151                         fault_reason, reason);
1152         else
1153                 printk(KERN_ERR
1154                        "DMAR:[%s] Request device [%02x:%02x.%d] "
1155                        "fault addr %llx \n"
1156                        "DMAR:[fault reason %02d] %s\n",
1157                        (type ? "DMA Read" : "DMA Write"),
1158                        (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1159                        PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1160         return 0;
1161 }
1162
1163 #define PRIMARY_FAULT_REG_LEN (16)
1164 irqreturn_t dmar_fault(int irq, void *dev_id)
1165 {
1166         struct intel_iommu *iommu = dev_id;
1167         int reg, fault_index;
1168         u32 fault_status;
1169         unsigned long flag;
1170
1171         spin_lock_irqsave(&iommu->register_lock, flag);
1172         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1173         if (fault_status)
1174                 printk(KERN_ERR "DRHD: handling fault status reg %x\n",
1175                        fault_status);
1176
1177         /* TBD: ignore advanced fault log currently */
1178         if (!(fault_status & DMA_FSTS_PPF))
1179                 goto clear_rest;
1180
1181         fault_index = dma_fsts_fault_record_index(fault_status);
1182         reg = cap_fault_reg_offset(iommu->cap);
1183         while (1) {
1184                 u8 fault_reason;
1185                 u16 source_id;
1186                 u64 guest_addr;
1187                 int type;
1188                 u32 data;
1189
1190                 /* highest 32 bits */
1191                 data = readl(iommu->reg + reg +
1192                                 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1193                 if (!(data & DMA_FRCD_F))
1194                         break;
1195
1196                 fault_reason = dma_frcd_fault_reason(data);
1197                 type = dma_frcd_type(data);
1198
1199                 data = readl(iommu->reg + reg +
1200                                 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1201                 source_id = dma_frcd_source_id(data);
1202
1203                 guest_addr = dmar_readq(iommu->reg + reg +
1204                                 fault_index * PRIMARY_FAULT_REG_LEN);
1205                 guest_addr = dma_frcd_page_addr(guest_addr);
1206                 /* clear the fault */
1207                 writel(DMA_FRCD_F, iommu->reg + reg +
1208                         fault_index * PRIMARY_FAULT_REG_LEN + 12);
1209
1210                 spin_unlock_irqrestore(&iommu->register_lock, flag);
1211
1212                 dmar_fault_do_one(iommu, type, fault_reason,
1213                                 source_id, guest_addr);
1214
1215                 fault_index++;
1216                 if (fault_index > cap_num_fault_regs(iommu->cap))
1217                         fault_index = 0;
1218                 spin_lock_irqsave(&iommu->register_lock, flag);
1219         }
1220 clear_rest:
1221         /* clear all the other faults */
1222         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1223         writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1224
1225         spin_unlock_irqrestore(&iommu->register_lock, flag);
1226         return IRQ_HANDLED;
1227 }
1228
1229 int dmar_set_interrupt(struct intel_iommu *iommu)
1230 {
1231         int irq, ret;
1232
1233         /*
1234          * Check if the fault interrupt is already initialized.
1235          */
1236         if (iommu->irq)
1237                 return 0;
1238
1239         irq = create_irq();
1240         if (!irq) {
1241                 printk(KERN_ERR "IOMMU: no free vectors\n");
1242                 return -EINVAL;
1243         }
1244
1245         set_irq_data(irq, iommu);
1246         iommu->irq = irq;
1247
1248         ret = arch_setup_dmar_msi(irq);
1249         if (ret) {
1250                 set_irq_data(irq, NULL);
1251                 iommu->irq = 0;
1252                 destroy_irq(irq);
1253                 return ret;
1254         }
1255
1256         ret = request_irq(irq, dmar_fault, 0, iommu->name, iommu);
1257         if (ret)
1258                 printk(KERN_ERR "IOMMU: can't request irq\n");
1259         return ret;
1260 }
1261
1262 int __init enable_drhd_fault_handling(void)
1263 {
1264         struct dmar_drhd_unit *drhd;
1265
1266         /*
1267          * Enable fault control interrupt.
1268          */
1269         for_each_drhd_unit(drhd) {
1270                 int ret;
1271                 struct intel_iommu *iommu = drhd->iommu;
1272                 ret = dmar_set_interrupt(iommu);
1273
1274                 if (ret) {
1275                         printk(KERN_ERR "DRHD %Lx: failed to enable fault, "
1276                                " interrupt, ret %d\n",
1277                                (unsigned long long)drhd->reg_base_addr, ret);
1278                         return -1;
1279                 }
1280         }
1281
1282         return 0;
1283 }
1284
1285 /*
1286  * Re-enable Queued Invalidation interface.
1287  */
1288 int dmar_reenable_qi(struct intel_iommu *iommu)
1289 {
1290         if (!ecap_qis(iommu->ecap))
1291                 return -ENOENT;
1292
1293         if (!iommu->qi)
1294                 return -ENOENT;
1295
1296         /*
1297          * First disable queued invalidation.
1298          */
1299         dmar_disable_qi(iommu);
1300         /*
1301          * Then enable queued invalidation again. Since there is no pending
1302          * invalidation requests now, it's safe to re-enable queued
1303          * invalidation.
1304          */
1305         __dmar_enable_qi(iommu);
1306
1307         return 0;
1308 }