drivers/pci/dmar.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  *
  22  * This file implements early detection/parsing of Remapping Devices
  23  * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  24  * tables.
  25  *
  26  * These routines are used by both DMA-remapping and Interrupt-remapping
  27  */
  28
  29 #include <linux/pci.h>
  30 #include <linux/dmar.h>
  31 #include <linux/iova.h>
  32 #include <linux/intel-iommu.h>
  33 #include <linux/timer.h>
  34 #include <linux/irq.h>
  35 #include <linux/interrupt.h>
  36 #include <linux/tboot.h>
  37
  38 #define PREFIX "DMAR: "
  39
  40 /* No locks are needed as DMA remapping hardware unit
  41  * list is constructed at boot time and hotplug of
  42  * these units are not supported by the architecture.
  43  */
  44 LIST_HEAD(dmar_drhd_units);
  45
  46 static struct acpi_table_header * __initdata dmar_tbl;
  47 static acpi_size dmar_tbl_size;
  48
  49 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  50 {
  51         /*
  52          * add INCLUDE_ALL at the tail, so scan the list will find it at
  53          * the very end.
  54          */
  55         if (drhd->include_all)
  56                 list_add_tail(&drhd->list, &dmar_drhd_units);
  57         else
  58                 list_add(&drhd->list, &dmar_drhd_units);
  59 }
  60
  61 static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
  62                                            struct pci_dev **dev, u16 segment)
  63 {
  64         struct pci_bus *bus;
  65         struct pci_dev *pdev = NULL;
  66         struct acpi_dmar_pci_path *path;
  67         int count;
  68
  69         bus = pci_find_bus(segment, scope->bus);
  70         path = (struct acpi_dmar_pci_path *)(scope + 1);
  71         count = (scope->length - sizeof(struct acpi_dmar_device_scope))
  72                 / sizeof(struct acpi_dmar_pci_path);
  73
  74         while (count) {
  75                 if (pdev)
  76                         pci_dev_put(pdev);
  77                 /*
  78                  * Some BIOSes list non-exist devices in DMAR table, just
  79                  * ignore it
  80                  */
  81                 if (!bus) {
  82                         printk(KERN_WARNING
  83                         PREFIX "Device scope bus [%d] not found\n",
  84                         scope->bus);
  85                         break;
  86                 }
  87                 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
  88                 if (!pdev) {
  89                         printk(KERN_WARNING PREFIX
  90                         "Device scope device [%04x:%02x:%02x.%02x] not found\n",
  91                                 segment, bus->number, path->dev, path->fn);
  92                         break;
  93                 }
  94                 path ++;
  95                 count --;
  96                 bus = pdev->subordinate;
  97         }
  98         if (!pdev) {
  99                 printk(KERN_WARNING PREFIX
 100                 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
 101                 segment, scope->bus, path->dev, path->fn);
 102                 *dev = NULL;
 103                 return 0;
 104         }
 105         if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
 106                         pdev->subordinate) || (scope->entry_type == \
 107                         ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
 108                 pci_dev_put(pdev);
 109                 printk(KERN_WARNING PREFIX
 110                         "Device scope type does not match for %s\n",
 111                          pci_name(pdev));
 112                 return -EINVAL;
 113         }
 114         *dev = pdev;
 115         return 0;
 116 }
 117
 118 static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
 119                                        struct pci_dev ***devices, u16 segment)
 120 {
 121         struct acpi_dmar_device_scope *scope;
 122         void * tmp = start;
 123         int index;
 124         int ret;
 125
 126         *cnt = 0;
 127         while (start < end) {
 128                 scope = start;
 129                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 130                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 131                         (*cnt)++;
 132                 else
 133                         printk(KERN_WARNING PREFIX
 134                                 "Unsupported device scope\n");
 135                 start += scope->length;
 136         }
 137         if (*cnt == 0)
 138                 return 0;
 139
 140         *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
 141         if (!*devices)
 142                 return -ENOMEM;
 143
 144         start = tmp;
 145         index = 0;
 146         while (start < end) {
 147                 scope = start;
 148                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 149                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
 150                         ret = dmar_parse_one_dev_scope(scope,
 151                                 &(*devices)[index], segment);
 152                         if (ret) {
 153                                 kfree(*devices);
 154                                 return ret;
 155                         }
 156                         index ++;
 157                 }
 158                 start += scope->length;
 159         }
 160
 161         return 0;
 162 }
 163
 164 /**
 165  * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 166  * structure which uniquely represent one DMA remapping hardware unit
 167  * present in the platform
 168  */
 169 static int __init
 170 dmar_parse_one_drhd(struct acpi_dmar_header *header)
 171 {
 172         struct acpi_dmar_hardware_unit *drhd;
 173         struct dmar_drhd_unit *dmaru;
 174         int ret = 0;
 175
 176         drhd = (struct acpi_dmar_hardware_unit *)header;
 177         if (!drhd->address) {
 178                 /* Promote an attitude of violence to a BIOS engineer today */
 179                 WARN(1, "Your BIOS is broken; DMAR reported at address zero!\n"
 180                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 181                      dmi_get_system_info(DMI_BIOS_VENDOR),
 182                      dmi_get_system_info(DMI_BIOS_VERSION),
 183                      dmi_get_system_info(DMI_PRODUCT_VERSION));
 184                 return -ENODEV;
 185         }
 186         dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
 187         if (!dmaru)
 188                 return -ENOMEM;
 189
 190         dmaru->hdr = header;
 191         dmaru->reg_base_addr = drhd->address;
 192         dmaru->segment = drhd->segment;
 193         dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
 194
 195         ret = alloc_iommu(dmaru);
 196         if (ret) {
 197                 kfree(dmaru);
 198                 return ret;
 199         }
 200         dmar_register_drhd_unit(dmaru);
 201         return 0;
 202 }
 203
 204 static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
 205 {
 206         struct acpi_dmar_hardware_unit *drhd;
 207         int ret = 0;
 208
 209         drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
 210
 211         if (dmaru->include_all)
 212                 return 0;
 213
 214         ret = dmar_parse_dev_scope((void *)(drhd + 1),
 215                                 ((void *)drhd) + drhd->header.length,
 216                                 &dmaru->devices_cnt, &dmaru->devices,
 217                                 drhd->segment);
 218         if (ret) {
 219                 list_del(&dmaru->list);
 220                 kfree(dmaru);
 221         }
 222         return ret;
 223 }
 224
 225 #ifdef CONFIG_DMAR
 226 LIST_HEAD(dmar_rmrr_units);
 227
 228 static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
 229 {
 230         list_add(&rmrr->list, &dmar_rmrr_units);
 231 }
 232
 233
 234 static int __init
 235 dmar_parse_one_rmrr(struct acpi_dmar_header *header)
 236 {
 237         struct acpi_dmar_reserved_memory *rmrr;
 238         struct dmar_rmrr_unit *rmrru;
 239
 240         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
 241         if (!rmrru)
 242                 return -ENOMEM;
 243
 244         rmrru->hdr = header;
 245         rmrr = (struct acpi_dmar_reserved_memory *)header;
 246         rmrru->base_address = rmrr->base_address;
 247         rmrru->end_address = rmrr->end_address;
 248
 249         dmar_register_rmrr_unit(rmrru);
 250         return 0;
 251 }
 252
 253 static int __init
 254 rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
 255 {
 256         struct acpi_dmar_reserved_memory *rmrr;
 257         int ret;
 258
 259         rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
 260         ret = dmar_parse_dev_scope((void *)(rmrr + 1),
 261                 ((void *)rmrr) + rmrr->header.length,
 262                 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
 263
 264         if (ret || (rmrru->devices_cnt == 0)) {
 265                 list_del(&rmrru->list);
 266                 kfree(rmrru);
 267         }
 268         return ret;
 269 }
 270
 271 static LIST_HEAD(dmar_atsr_units);
 272
 273 static int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
 274 {
 275         struct acpi_dmar_atsr *atsr;
 276         struct dmar_atsr_unit *atsru;
 277
 278         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
 279         atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
 280         if (!atsru)
 281                 return -ENOMEM;
 282
 283         atsru->hdr = hdr;
 284         atsru->include_all = atsr->flags & 0x1;
 285
 286         list_add(&atsru->list, &dmar_atsr_units);
 287
 288         return 0;
 289 }
 290
 291 static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
 292 {
 293         int rc;
 294         struct acpi_dmar_atsr *atsr;
 295
 296         if (atsru->include_all)
 297                 return 0;
 298
 299         atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
 300         rc = dmar_parse_dev_scope((void *)(atsr + 1),
 301                                 (void *)atsr + atsr->header.length,
 302                                 &atsru->devices_cnt, &atsru->devices,
 303                                 atsr->segment);
 304         if (rc || !atsru->devices_cnt) {
 305                 list_del(&atsru->list);
 306                 kfree(atsru);
 307         }
 308
 309         return rc;
 310 }
 311
 312 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
 313 {
 314         int i;
 315         struct pci_bus *bus;
 316         struct acpi_dmar_atsr *atsr;
 317         struct dmar_atsr_unit *atsru;
 318
 319         list_for_each_entry(atsru, &dmar_atsr_units, list) {
 320                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
 321                 if (atsr->segment == pci_domain_nr(dev->bus))
 322                         goto found;
 323         }
 324
 325         return 0;
 326
 327 found:
 328         for (bus = dev->bus; bus; bus = bus->parent) {
 329                 struct pci_dev *bridge = bus->self;
 330
 331                 if (!bridge || !bridge->is_pcie ||
 332                     bridge->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
 333                         return 0;
 334
 335                 if (bridge->pcie_type == PCI_EXP_TYPE_ROOT_PORT) {
 336                         for (i = 0; i < atsru->devices_cnt; i++)
 337                                 if (atsru->devices[i] == bridge)
 338                                         return 1;
 339                         break;
 340                 }
 341         }
 342
 343         if (atsru->include_all)
 344                 return 1;
 345
 346         return 0;
 347 }
 348 #endif
 349
 350 static void __init
 351 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
 352 {
 353         struct acpi_dmar_hardware_unit *drhd;
 354         struct acpi_dmar_reserved_memory *rmrr;
 355         struct acpi_dmar_atsr *atsr;
 356
 357         switch (header->type) {
 358         case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 359                 drhd = container_of(header, struct acpi_dmar_hardware_unit,
 360                                     header);
 361                 printk (KERN_INFO PREFIX
 362                         "DRHD base: %#016Lx flags: %#x\n",
 363                         (unsigned long long)drhd->address, drhd->flags);
 364                 break;
 365         case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 366                 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
 367                                     header);
 368                 printk (KERN_INFO PREFIX
 369                         "RMRR base: %#016Lx end: %#016Lx\n",
 370                         (unsigned long long)rmrr->base_address,
 371                         (unsigned long long)rmrr->end_address);
 372                 break;
 373         case ACPI_DMAR_TYPE_ATSR:
 374                 atsr = container_of(header, struct acpi_dmar_atsr, header);
 375                 printk(KERN_INFO PREFIX "ATSR flags: %#x\n", atsr->flags);
 376                 break;
 377         }
 378 }
 379
 380 /**
 381  * dmar_table_detect - checks to see if the platform supports DMAR devices
 382  */
 383 static int __init dmar_table_detect(void)
 384 {
 385         acpi_status status = AE_OK;
 386
 387         /* if we could find DMAR table, then there are DMAR devices */
 388         status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
 389                                 (struct acpi_table_header **)&dmar_tbl,
 390                                 &dmar_tbl_size);
 391
 392         if (ACPI_SUCCESS(status) && !dmar_tbl) {
 393                 printk (KERN_WARNING PREFIX "Unable to map DMAR\n");
 394                 status = AE_NOT_FOUND;
 395         }
 396
 397         return (ACPI_SUCCESS(status) ? 1 : 0);
 398 }
 399
 400 /**
 401  * parse_dmar_table - parses the DMA reporting table
 402  */
 403 static int __init
 404 parse_dmar_table(void)
 405 {
 406         struct acpi_table_dmar *dmar;
 407         struct acpi_dmar_header *entry_header;
 408         int ret = 0;
 409
 410         /*
 411          * Do it again, earlier dmar_tbl mapping could be mapped with
 412          * fixed map.
 413          */
 414         dmar_table_detect();
 415
 416         /*
 417          * ACPI tables may not be DMA protected by tboot, so use DMAR copy
 418          * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
 419          */
 420         dmar_tbl = tboot_get_dmar_table(dmar_tbl);
 421
 422         dmar = (struct acpi_table_dmar *)dmar_tbl;
 423         if (!dmar)
 424                 return -ENODEV;
 425
 426         if (dmar->width < PAGE_SHIFT - 1) {
 427                 printk(KERN_WARNING PREFIX "Invalid DMAR haw\n");
 428                 return -EINVAL;
 429         }
 430
 431         printk (KERN_INFO PREFIX "Host address width %d\n",
 432                 dmar->width + 1);
 433
 434         entry_header = (struct acpi_dmar_header *)(dmar + 1);
 435         while (((unsigned long)entry_header) <
 436                         (((unsigned long)dmar) + dmar_tbl->length)) {
 437                 /* Avoid looping forever on bad ACPI tables */
 438                 if (entry_header->length == 0) {
 439                         printk(KERN_WARNING PREFIX
 440                                 "Invalid 0-length structure\n");
 441                         ret = -EINVAL;
 442                         break;
 443                 }
 444
 445                 dmar_table_print_dmar_entry(entry_header);
 446
 447                 switch (entry_header->type) {
 448                 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 449                         ret = dmar_parse_one_drhd(entry_header);
 450                         break;
 451                 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 452 #ifdef CONFIG_DMAR
 453                         ret = dmar_parse_one_rmrr(entry_header);
 454 #endif
 455                         break;
 456                 case ACPI_DMAR_TYPE_ATSR:
 457 #ifdef CONFIG_DMAR
 458                         ret = dmar_parse_one_atsr(entry_header);
 459 #endif
 460                         break;
 461                 default:
 462                         printk(KERN_WARNING PREFIX
 463                                 "Unknown DMAR structure type\n");
 464                         ret = 0; /* for forward compatibility */
 465                         break;
 466                 }
 467                 if (ret)
 468                         break;
 469
 470                 entry_header = ((void *)entry_header + entry_header->length);
 471         }
 472         return ret;
 473 }
 474
 475 int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
 476                           struct pci_dev *dev)
 477 {
 478         int index;
 479
 480         while (dev) {
 481                 for (index = 0; index < cnt; index++)
 482                         if (dev == devices[index])
 483                                 return 1;
 484
 485                 /* Check our parent */
 486                 dev = dev->bus->self;
 487         }
 488
 489         return 0;
 490 }
 491
 492 struct dmar_drhd_unit *
 493 dmar_find_matched_drhd_unit(struct pci_dev *dev)
 494 {
 495         struct dmar_drhd_unit *dmaru = NULL;
 496         struct acpi_dmar_hardware_unit *drhd;
 497
 498         list_for_each_entry(dmaru, &dmar_drhd_units, list) {
 499                 drhd = container_of(dmaru->hdr,
 500                                     struct acpi_dmar_hardware_unit,
 501                                     header);
 502
 503                 if (dmaru->include_all &&
 504                     drhd->segment == pci_domain_nr(dev->bus))
 505                         return dmaru;
 506
 507                 if (dmar_pci_device_match(dmaru->devices,
 508                                           dmaru->devices_cnt, dev))
 509                         return dmaru;
 510         }
 511
 512         return NULL;
 513 }
 514
 515 int __init dmar_dev_scope_init(void)
 516 {
 517         struct dmar_drhd_unit *drhd, *drhd_n;
 518         int ret = -ENODEV;
 519
 520         list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
 521                 ret = dmar_parse_dev(drhd);
 522                 if (ret)
 523                         return ret;
 524         }
 525
 526 #ifdef CONFIG_DMAR
 527         {
 528                 struct dmar_rmrr_unit *rmrr, *rmrr_n;
 529                 struct dmar_atsr_unit *atsr, *atsr_n;
 530
 531                 list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
 532                         ret = rmrr_parse_dev(rmrr);
 533                         if (ret)
 534                                 return ret;
 535                 }
 536
 537                 list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
 538                         ret = atsr_parse_dev(atsr);
 539                         if (ret)
 540                                 return ret;
 541                 }
 542         }
 543 #endif
 544
 545         return ret;
 546 }
 547
 548
 549 int __init dmar_table_init(void)
 550 {
 551         static int dmar_table_initialized;
 552         int ret;
 553
 554         if (dmar_table_initialized)
 555                 return 0;
 556
 557         dmar_table_initialized = 1;
 558
 559         ret = parse_dmar_table();
 560         if (ret) {
 561                 if (ret != -ENODEV)
 562                         printk(KERN_INFO PREFIX "parse DMAR table failure.\n");
 563                 return ret;
 564         }
 565
 566         if (list_empty(&dmar_drhd_units)) {
 567                 printk(KERN_INFO PREFIX "No DMAR devices found\n");
 568                 return -ENODEV;
 569         }
 570
 571 #ifdef CONFIG_DMAR
 572         if (list_empty(&dmar_rmrr_units))
 573                 printk(KERN_INFO PREFIX "No RMRR found\n");
 574
 575         if (list_empty(&dmar_atsr_units))
 576                 printk(KERN_INFO PREFIX "No ATSR found\n");
 577 #endif
 578
 579 #ifdef CONFIG_INTR_REMAP
 580         parse_ioapics_under_ir();
 581 #endif
 582         return 0;
 583 }
 584
 585 void __init detect_intel_iommu(void)
 586 {
 587         int ret;
 588
 589         ret = dmar_table_detect();
 590
 591         {
 592 #ifdef CONFIG_INTR_REMAP
 593                 struct acpi_table_dmar *dmar;
 594                 /*
 595                  * for now we will disable dma-remapping when interrupt
 596                  * remapping is enabled.
 597                  * When support for queued invalidation for IOTLB invalidation
 598                  * is added, we will not need this any more.
 599                  */
 600                 dmar = (struct acpi_table_dmar *) dmar_tbl;
 601                 if (ret && cpu_has_x2apic && dmar->flags & 0x1)
 602                         printk(KERN_INFO
 603                                "Queued invalidation will be enabled to support "
 604                                "x2apic and Intr-remapping.\n");
 605 #endif
 606 #ifdef CONFIG_DMAR
 607                 if (ret && !no_iommu && !iommu_detected && !swiotlb &&
 608                     !dmar_disabled)
 609                         iommu_detected = 1;
 610 #endif
 611         }
 612         early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
 613         dmar_tbl = NULL;
 614 }
 615
 616
 617 int alloc_iommu(struct dmar_drhd_unit *drhd)
 618 {
 619         struct intel_iommu *iommu;
 620         int map_size;
 621         u32 ver;
 622         static int iommu_allocated = 0;
 623         int agaw = 0;
 624         int msagaw = 0;
 625
 626         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
 627         if (!iommu)
 628                 return -ENOMEM;
 629
 630         iommu->seq_id = iommu_allocated++;
 631         sprintf (iommu->name, "dmar%d", iommu->seq_id);
 632
 633         iommu->reg = ioremap(drhd->reg_base_addr, VTD_PAGE_SIZE);
 634         if (!iommu->reg) {
 635                 printk(KERN_ERR "IOMMU: can't map the region\n");
 636                 goto error;
 637         }
 638         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 639         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 640
 641 #ifdef CONFIG_DMAR
 642         agaw = iommu_calculate_agaw(iommu);
 643         if (agaw < 0) {
 644                 printk(KERN_ERR
 645                        "Cannot get a valid agaw for iommu (seq_id = %d)\n",
 646                        iommu->seq_id);
 647                 goto error;
 648         }
 649         msagaw = iommu_calculate_max_sagaw(iommu);
 650         if (msagaw < 0) {
 651                 printk(KERN_ERR
 652                         "Cannot get a valid max agaw for iommu (seq_id = %d)\n",
 653                         iommu->seq_id);
 654                 goto error;
 655         }
 656 #endif
 657         iommu->agaw = agaw;
 658         iommu->msagaw = msagaw;
 659
 660         /* the registers might be more than one page */
 661         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 662                 cap_max_fault_reg_offset(iommu->cap));
 663         map_size = VTD_PAGE_ALIGN(map_size);
 664         if (map_size > VTD_PAGE_SIZE) {
 665                 iounmap(iommu->reg);
 666                 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
 667                 if (!iommu->reg) {
 668                         printk(KERN_ERR "IOMMU: can't map the region\n");
 669                         goto error;
 670                 }
 671         }
 672
 673         ver = readl(iommu->reg + DMAR_VER_REG);
 674         pr_debug("IOMMU %llx: ver %d:%d cap %llx ecap %llx\n",
 675                 (unsigned long long)drhd->reg_base_addr,
 676                 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
 677                 (unsigned long long)iommu->cap,
 678                 (unsigned long long)iommu->ecap);
 679
 680         spin_lock_init(&iommu->register_lock);
 681
 682         drhd->iommu = iommu;
 683         return 0;
 684 error:
 685         kfree(iommu);
 686         return -1;
 687 }
 688
 689 void free_iommu(struct intel_iommu *iommu)
 690 {
 691         if (!iommu)
 692                 return;
 693
 694 #ifdef CONFIG_DMAR
 695         free_dmar_iommu(iommu);
 696 #endif
 697
 698         if (iommu->reg)
 699                 iounmap(iommu->reg);
 700         kfree(iommu);
 701 }
 702
 703 /*
 704  * Reclaim all the submitted descriptors which have completed its work.
 705  */
 706 static inline void reclaim_free_desc(struct q_inval *qi)
 707 {
 708         while (qi->desc_status[qi->free_tail] == QI_DONE ||
 709                qi->desc_status[qi->free_tail] == QI_ABORT) {
 710                 qi->desc_status[qi->free_tail] = QI_FREE;
 711                 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
 712                 qi->free_cnt++;
 713         }
 714 }
 715
 716 static int qi_check_fault(struct intel_iommu *iommu, int index)
 717 {
 718         u32 fault;
 719         int head, tail;
 720         struct q_inval *qi = iommu->qi;
 721         int wait_index = (index + 1) % QI_LENGTH;
 722
 723         if (qi->desc_status[wait_index] == QI_ABORT)
 724                 return -EAGAIN;
 725
 726         fault = readl(iommu->reg + DMAR_FSTS_REG);
 727
 728         /*
 729          * If IQE happens, the head points to the descriptor associated
 730          * with the error. No new descriptors are fetched until the IQE
 731          * is cleared.
 732          */
 733         if (fault & DMA_FSTS_IQE) {
 734                 head = readl(iommu->reg + DMAR_IQH_REG);
 735                 if ((head >> DMAR_IQ_SHIFT) == index) {
 736                         printk(KERN_ERR "VT-d detected invalid descriptor: "
 737                                 "low=%llx, high=%llx\n",
 738                                 (unsigned long long)qi->desc[index].low,
 739                                 (unsigned long long)qi->desc[index].high);
 740                         memcpy(&qi->desc[index], &qi->desc[wait_index],
 741                                         sizeof(struct qi_desc));
 742                         __iommu_flush_cache(iommu, &qi->desc[index],
 743                                         sizeof(struct qi_desc));
 744                         writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
 745                         return -EINVAL;
 746                 }
 747         }
 748
 749         /*
 750          * If ITE happens, all pending wait_desc commands are aborted.
 751          * No new descriptors are fetched until the ITE is cleared.
 752          */
 753         if (fault & DMA_FSTS_ITE) {
 754                 head = readl(iommu->reg + DMAR_IQH_REG);
 755                 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
 756                 head |= 1;
 757                 tail = readl(iommu->reg + DMAR_IQT_REG);
 758                 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
 759
 760                 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
 761
 762                 do {
 763                         if (qi->desc_status[head] == QI_IN_USE)
 764                                 qi->desc_status[head] = QI_ABORT;
 765                         head = (head - 2 + QI_LENGTH) % QI_LENGTH;
 766                 } while (head != tail);
 767
 768                 if (qi->desc_status[wait_index] == QI_ABORT)
 769                         return -EAGAIN;
 770         }
 771
 772         if (fault & DMA_FSTS_ICE)
 773                 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
 774
 775         return 0;
 776 }
 777
 778 /*
 779  * Submit the queued invalidation descriptor to the remapping
 780  * hardware unit and wait for its completion.
 781  */
 782 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
 783 {
 784         int rc;
 785         struct q_inval *qi = iommu->qi;
 786         struct qi_desc *hw, wait_desc;
 787         int wait_index, index;
 788         unsigned long flags;
 789
 790         if (!qi)
 791                 return 0;
 792
 793         hw = qi->desc;
 794
 795 restart:
 796         rc = 0;
 797
 798         spin_lock_irqsave(&qi->q_lock, flags);
 799         while (qi->free_cnt < 3) {
 800                 spin_unlock_irqrestore(&qi->q_lock, flags);
 801                 cpu_relax();
 802                 spin_lock_irqsave(&qi->q_lock, flags);
 803         }
 804
 805         index = qi->free_head;
 806         wait_index = (index + 1) % QI_LENGTH;
 807
 808         qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
 809
 810         hw[index] = *desc;
 811
 812         wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
 813                         QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
 814         wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
 815
 816         hw[wait_index] = wait_desc;
 817
 818         __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
 819         __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
 820
 821         qi->free_head = (qi->free_head + 2) % QI_LENGTH;
 822         qi->free_cnt -= 2;
 823
 824         /*
 825          * update the HW tail register indicating the presence of
 826          * new descriptors.
 827          */
 828         writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
 829
 830         while (qi->desc_status[wait_index] != QI_DONE) {
 831                 /*
 832                  * We will leave the interrupts disabled, to prevent interrupt
 833                  * context to queue another cmd while a cmd is already submitted
 834                  * and waiting for completion on this cpu. This is to avoid
 835                  * a deadlock where the interrupt context can wait indefinitely
 836                  * for free slots in the queue.
 837                  */
 838                 rc = qi_check_fault(iommu, index);
 839                 if (rc)
 840                         break;
 841
 842                 spin_unlock(&qi->q_lock);
 843                 cpu_relax();
 844                 spin_lock(&qi->q_lock);
 845         }
 846
 847         qi->desc_status[index] = QI_DONE;
 848
 849         reclaim_free_desc(qi);
 850         spin_unlock_irqrestore(&qi->q_lock, flags);
 851
 852         if (rc == -EAGAIN)
 853                 goto restart;
 854
 855         return rc;
 856 }
 857
 858 /*
 859  * Flush the global interrupt entry cache.
 860  */
 861 void qi_global_iec(struct intel_iommu *iommu)
 862 {
 863         struct qi_desc desc;
 864
 865         desc.low = QI_IEC_TYPE;
 866         desc.high = 0;
 867
 868         /* should never fail */
 869         qi_submit_sync(&desc, iommu);
 870 }
 871
 872 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
 873                       u64 type)
 874 {
 875         struct qi_desc desc;
 876
 877         desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
 878                         | QI_CC_GRAN(type) | QI_CC_TYPE;
 879         desc.high = 0;
 880
 881         qi_submit_sync(&desc, iommu);
 882 }
 883
 884 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
 885                     unsigned int size_order, u64 type)
 886 {
 887         u8 dw = 0, dr = 0;
 888
 889         struct qi_desc desc;
 890         int ih = 0;
 891
 892         if (cap_write_drain(iommu->cap))
 893                 dw = 1;
 894
 895         if (cap_read_drain(iommu->cap))
 896                 dr = 1;
 897
 898         desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
 899                 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
 900         desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
 901                 | QI_IOTLB_AM(size_order);
 902
 903         qi_submit_sync(&desc, iommu);
 904 }
 905
 906 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
 907                         u64 addr, unsigned mask)
 908 {
 909         struct qi_desc desc;
 910
 911         if (mask) {
 912                 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
 913                 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
 914                 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
 915         } else
 916                 desc.high = QI_DEV_IOTLB_ADDR(addr);
 917
 918         if (qdep >= QI_DEV_IOTLB_MAX_INVS)
 919                 qdep = 0;
 920
 921         desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
 922                    QI_DIOTLB_TYPE;
 923
 924         qi_submit_sync(&desc, iommu);
 925 }
 926
 927 /*
 928  * Disable Queued Invalidation interface.
 929  */
 930 void dmar_disable_qi(struct intel_iommu *iommu)
 931 {
 932         unsigned long flags;
 933         u32 sts;
 934         cycles_t start_time = get_cycles();
 935
 936         if (!ecap_qis(iommu->ecap))
 937                 return;
 938
 939         spin_lock_irqsave(&iommu->register_lock, flags);
 940
 941         sts =  dmar_readq(iommu->reg + DMAR_GSTS_REG);
 942         if (!(sts & DMA_GSTS_QIES))
 943                 goto end;
 944
 945         /*
 946          * Give a chance to HW to complete the pending invalidation requests.
 947          */
 948         while ((readl(iommu->reg + DMAR_IQT_REG) !=
 949                 readl(iommu->reg + DMAR_IQH_REG)) &&
 950                 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
 951                 cpu_relax();
 952
 953         iommu->gcmd &= ~DMA_GCMD_QIE;
 954         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
 955
 956         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
 957                       !(sts & DMA_GSTS_QIES), sts);
 958 end:
 959         spin_unlock_irqrestore(&iommu->register_lock, flags);
 960 }
 961
 962 /*
 963  * Enable queued invalidation.
 964  */
 965 static void __dmar_enable_qi(struct intel_iommu *iommu)
 966 {
 967         u32 sts;
 968         unsigned long flags;
 969         struct q_inval *qi = iommu->qi;
 970
 971         qi->free_head = qi->free_tail = 0;
 972         qi->free_cnt = QI_LENGTH;
 973
 974         spin_lock_irqsave(&iommu->register_lock, flags);
 975
 976         /* write zero to the tail reg */
 977         writel(0, iommu->reg + DMAR_IQT_REG);
 978
 979         dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
 980
 981         iommu->gcmd |= DMA_GCMD_QIE;
 982         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
 983
 984         /* Make sure hardware complete it */
 985         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
 986
 987         spin_unlock_irqrestore(&iommu->register_lock, flags);
 988 }
 989
 990 /*
 991  * Enable Queued Invalidation interface. This is a must to support
 992  * interrupt-remapping. Also used by DMA-remapping, which replaces
 993  * register based IOTLB invalidation.
 994  */
 995 int dmar_enable_qi(struct intel_iommu *iommu)
 996 {
 997         struct q_inval *qi;
 998
 999         if (!ecap_qis(iommu->ecap))
1000                 return -ENOENT;
1001
1002         /*
1003          * queued invalidation is already setup and enabled.
1004          */
1005         if (iommu->qi)
1006                 return 0;
1007
1008         iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1009         if (!iommu->qi)
1010                 return -ENOMEM;
1011
1012         qi = iommu->qi;
1013
1014         qi->desc = (void *)(get_zeroed_page(GFP_ATOMIC));
1015         if (!qi->desc) {
1016                 kfree(qi);
1017                 iommu->qi = 0;
1018                 return -ENOMEM;
1019         }
1020
1021         qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1022         if (!qi->desc_status) {
1023                 free_page((unsigned long) qi->desc);
1024                 kfree(qi);
1025                 iommu->qi = 0;
1026                 return -ENOMEM;
1027         }
1028
1029         qi->free_head = qi->free_tail = 0;
1030         qi->free_cnt = QI_LENGTH;
1031
1032         spin_lock_init(&qi->q_lock);
1033
1034         __dmar_enable_qi(iommu);
1035
1036         return 0;
1037 }
1038
1039 /* iommu interrupt handling. Most stuff are MSI-like. */
1040
1041 enum faulttype {
1042         DMA_REMAP,
1043         INTR_REMAP,
1044         UNKNOWN,
1045 };
1046
1047 static const char *dma_remap_fault_reasons[] =
1048 {
1049         "Software",
1050         "Present bit in root entry is clear",
1051         "Present bit in context entry is clear",
1052         "Invalid context entry",
1053         "Access beyond MGAW",
1054         "PTE Write access is not set",
1055         "PTE Read access is not set",
1056         "Next page table ptr is invalid",
1057         "Root table address invalid",
1058         "Context table ptr is invalid",
1059         "non-zero reserved fields in RTP",
1060         "non-zero reserved fields in CTP",
1061         "non-zero reserved fields in PTE",
1062 };
1063
1064 static const char *intr_remap_fault_reasons[] =
1065 {
1066         "Detected reserved fields in the decoded interrupt-remapped request",
1067         "Interrupt index exceeded the interrupt-remapping table size",
1068         "Present field in the IRTE entry is clear",
1069         "Error accessing interrupt-remapping table pointed by IRTA_REG",
1070         "Detected reserved fields in the IRTE entry",
1071         "Blocked a compatibility format interrupt request",
1072         "Blocked an interrupt request due to source-id verification failure",
1073 };
1074
1075 #define MAX_FAULT_REASON_IDX    (ARRAY_SIZE(fault_reason_strings) - 1)
1076
1077 const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1078 {
1079         if (fault_reason >= 0x20 && (fault_reason <= 0x20 +
1080                                      ARRAY_SIZE(intr_remap_fault_reasons))) {
1081                 *fault_type = INTR_REMAP;
1082                 return intr_remap_fault_reasons[fault_reason - 0x20];
1083         } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1084                 *fault_type = DMA_REMAP;
1085                 return dma_remap_fault_reasons[fault_reason];
1086         } else {
1087                 *fault_type = UNKNOWN;
1088                 return "Unknown";
1089         }
1090 }
1091
1092 void dmar_msi_unmask(unsigned int irq)
1093 {
1094         struct intel_iommu *iommu = get_irq_data(irq);
1095         unsigned long flag;
1096
1097         /* unmask it */
1098         spin_lock_irqsave(&iommu->register_lock, flag);
1099         writel(0, iommu->reg + DMAR_FECTL_REG);
1100         /* Read a reg to force flush the post write */
1101         readl(iommu->reg + DMAR_FECTL_REG);
1102         spin_unlock_irqrestore(&iommu->register_lock, flag);
1103 }
1104
1105 void dmar_msi_mask(unsigned int irq)
1106 {
1107         unsigned long flag;
1108         struct intel_iommu *iommu = get_irq_data(irq);
1109
1110         /* mask it */
1111         spin_lock_irqsave(&iommu->register_lock, flag);
1112         writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1113         /* Read a reg to force flush the post write */
1114         readl(iommu->reg + DMAR_FECTL_REG);
1115         spin_unlock_irqrestore(&iommu->register_lock, flag);
1116 }
1117
1118 void dmar_msi_write(int irq, struct msi_msg *msg)
1119 {
1120         struct intel_iommu *iommu = get_irq_data(irq);
1121         unsigned long flag;
1122
1123         spin_lock_irqsave(&iommu->register_lock, flag);
1124         writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1125         writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1126         writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1127         spin_unlock_irqrestore(&iommu->register_lock, flag);
1128 }
1129
1130 void dmar_msi_read(int irq, struct msi_msg *msg)
1131 {
1132         struct intel_iommu *iommu = get_irq_data(irq);
1133         unsigned long flag;
1134
1135         spin_lock_irqsave(&iommu->register_lock, flag);
1136         msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1137         msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1138         msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1139         spin_unlock_irqrestore(&iommu->register_lock, flag);
1140 }
1141
1142 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1143                 u8 fault_reason, u16 source_id, unsigned long long addr)
1144 {
1145         const char *reason;
1146         int fault_type;
1147
1148         reason = dmar_get_fault_reason(fault_reason, &fault_type);
1149
1150         if (fault_type == INTR_REMAP)
1151                 printk(KERN_ERR "INTR-REMAP: Request device [[%02x:%02x.%d] "
1152                        "fault index %llx\n"
1153                         "INTR-REMAP:[fault reason %02d] %s\n",
1154                         (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1155                         PCI_FUNC(source_id & 0xFF), addr >> 48,
1156                         fault_reason, reason);
1157         else
1158                 printk(KERN_ERR
1159                        "DMAR:[%s] Request device [%02x:%02x.%d] "
1160                        "fault addr %llx \n"
1161                        "DMAR:[fault reason %02d] %s\n",
1162                        (type ? "DMA Read" : "DMA Write"),
1163                        (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1164                        PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1165         return 0;
1166 }
1167
1168 #define PRIMARY_FAULT_REG_LEN (16)
1169 irqreturn_t dmar_fault(int irq, void *dev_id)
1170 {
1171         struct intel_iommu *iommu = dev_id;
1172         int reg, fault_index;
1173         u32 fault_status;
1174         unsigned long flag;
1175
1176         spin_lock_irqsave(&iommu->register_lock, flag);
1177         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1178         if (fault_status)
1179                 printk(KERN_ERR "DRHD: handling fault status reg %x\n",
1180                        fault_status);
1181
1182         /* TBD: ignore advanced fault log currently */
1183         if (!(fault_status & DMA_FSTS_PPF))
1184                 goto clear_rest;
1185
1186         fault_index = dma_fsts_fault_record_index(fault_status);
1187         reg = cap_fault_reg_offset(iommu->cap);
1188         while (1) {
1189                 u8 fault_reason;
1190                 u16 source_id;
1191                 u64 guest_addr;
1192                 int type;
1193                 u32 data;
1194
1195                 /* highest 32 bits */
1196                 data = readl(iommu->reg + reg +
1197                                 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1198                 if (!(data & DMA_FRCD_F))
1199                         break;
1200
1201                 fault_reason = dma_frcd_fault_reason(data);
1202                 type = dma_frcd_type(data);
1203
1204                 data = readl(iommu->reg + reg +
1205                                 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1206                 source_id = dma_frcd_source_id(data);
1207
1208                 guest_addr = dmar_readq(iommu->reg + reg +
1209                                 fault_index * PRIMARY_FAULT_REG_LEN);
1210                 guest_addr = dma_frcd_page_addr(guest_addr);
1211                 /* clear the fault */
1212                 writel(DMA_FRCD_F, iommu->reg + reg +
1213                         fault_index * PRIMARY_FAULT_REG_LEN + 12);
1214
1215                 spin_unlock_irqrestore(&iommu->register_lock, flag);
1216
1217                 dmar_fault_do_one(iommu, type, fault_reason,
1218                                 source_id, guest_addr);
1219
1220                 fault_index++;
1221                 if (fault_index > cap_num_fault_regs(iommu->cap))
1222                         fault_index = 0;
1223                 spin_lock_irqsave(&iommu->register_lock, flag);
1224         }
1225 clear_rest:
1226         /* clear all the other faults */
1227         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1228         writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1229
1230         spin_unlock_irqrestore(&iommu->register_lock, flag);
1231         return IRQ_HANDLED;
1232 }
1233
1234 int dmar_set_interrupt(struct intel_iommu *iommu)
1235 {
1236         int irq, ret;
1237
1238         /*
1239          * Check if the fault interrupt is already initialized.
1240          */
1241         if (iommu->irq)
1242                 return 0;
1243
1244         irq = create_irq();
1245         if (!irq) {
1246                 printk(KERN_ERR "IOMMU: no free vectors\n");
1247                 return -EINVAL;
1248         }
1249
1250         set_irq_data(irq, iommu);
1251         iommu->irq = irq;
1252
1253         ret = arch_setup_dmar_msi(irq);
1254         if (ret) {
1255                 set_irq_data(irq, NULL);
1256                 iommu->irq = 0;
1257                 destroy_irq(irq);
1258                 return ret;
1259         }
1260
1261         ret = request_irq(irq, dmar_fault, 0, iommu->name, iommu);
1262         if (ret)
1263                 printk(KERN_ERR "IOMMU: can't request irq\n");
1264         return ret;
1265 }
1266
1267 int __init enable_drhd_fault_handling(void)
1268 {
1269         struct dmar_drhd_unit *drhd;
1270
1271         /*
1272          * Enable fault control interrupt.
1273          */
1274         for_each_drhd_unit(drhd) {
1275                 int ret;
1276                 struct intel_iommu *iommu = drhd->iommu;
1277                 ret = dmar_set_interrupt(iommu);
1278
1279                 if (ret) {
1280                         printk(KERN_ERR "DRHD %Lx: failed to enable fault, "
1281                                " interrupt, ret %d\n",
1282                                (unsigned long long)drhd->reg_base_addr, ret);
1283                         return -1;
1284                 }
1285         }
1286
1287         return 0;
1288 }
1289
1290 /*
1291  * Re-enable Queued Invalidation interface.
1292  */
1293 int dmar_reenable_qi(struct intel_iommu *iommu)
1294 {
1295         if (!ecap_qis(iommu->ecap))
1296                 return -ENOENT;
1297
1298         if (!iommu->qi)
1299                 return -ENOENT;
1300
1301         /*
1302          * First disable queued invalidation.
1303          */
1304         dmar_disable_qi(iommu);
1305         /*
1306          * Then enable queued invalidation again. Since there is no pending
1307          * invalidation requests now, it's safe to re-enable queued
1308          * invalidation.
1309          */
1310         __dmar_enable_qi(iommu);
1311
1312         return 0;
1313 }