2 * Copyright (c) 2006, Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
22 * This file implements early detection/parsing of Remapping Devices
23 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
26 * These routines are used by both DMA-remapping and Interrupt-remapping
29 #include <linux/pci.h>
30 #include <linux/dmar.h>
31 #include <linux/iova.h>
32 #include <linux/intel-iommu.h>
33 #include <linux/timer.h>
34 #include <linux/irq.h>
35 #include <linux/interrupt.h>
36 #include <linux/tboot.h>
37 #include <linux/dmi.h>
38 #include <linux/slab.h>
39 #include <asm/iommu_table.h>
41 #define PREFIX "DMAR: "
43 /* No locks are needed as DMA remapping hardware unit
44 * list is constructed at boot time and hotplug of
45 * these units are not supported by the architecture.
47 LIST_HEAD(dmar_drhd_units);
49 struct acpi_table_header * __initdata dmar_tbl;
50 static acpi_size dmar_tbl_size;
52 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
55 * add INCLUDE_ALL at the tail, so scan the list will find it at
58 if (drhd->include_all)
59 list_add_tail(&drhd->list, &dmar_drhd_units);
61 list_add(&drhd->list, &dmar_drhd_units);
64 static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
65 struct pci_dev **dev, u16 segment)
68 struct pci_dev *pdev = NULL;
69 struct acpi_dmar_pci_path *path;
72 bus = pci_find_bus(segment, scope->bus);
73 path = (struct acpi_dmar_pci_path *)(scope + 1);
74 count = (scope->length - sizeof(struct acpi_dmar_device_scope))
75 / sizeof(struct acpi_dmar_pci_path);
81 * Some BIOSes list non-exist devices in DMAR table, just
86 PREFIX "Device scope bus [%d] not found\n",
90 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
92 printk(KERN_WARNING PREFIX
93 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
94 segment, bus->number, path->dev, path->fn);
99 bus = pdev->subordinate;
102 printk(KERN_WARNING PREFIX
103 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
104 segment, scope->bus, path->dev, path->fn);
108 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
109 pdev->subordinate) || (scope->entry_type == \
110 ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
112 printk(KERN_WARNING PREFIX
113 "Device scope type does not match for %s\n",
121 int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
122 struct pci_dev ***devices, u16 segment)
124 struct acpi_dmar_device_scope *scope;
130 while (start < end) {
132 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
133 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
135 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC) {
136 printk(KERN_WARNING PREFIX
137 "Unsupported device scope\n");
139 start += scope->length;
144 *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
150 while (start < end) {
152 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
153 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
154 ret = dmar_parse_one_dev_scope(scope,
155 &(*devices)[index], segment);
162 start += scope->length;
169 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
170 * structure which uniquely represent one DMA remapping hardware unit
171 * present in the platform
174 dmar_parse_one_drhd(struct acpi_dmar_header *header)
176 struct acpi_dmar_hardware_unit *drhd;
177 struct dmar_drhd_unit *dmaru;
180 drhd = (struct acpi_dmar_hardware_unit *)header;
181 dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
186 dmaru->reg_base_addr = drhd->address;
187 dmaru->segment = drhd->segment;
188 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
190 ret = alloc_iommu(dmaru);
195 dmar_register_drhd_unit(dmaru);
199 static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
201 struct acpi_dmar_hardware_unit *drhd;
204 drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
206 if (dmaru->include_all)
209 ret = dmar_parse_dev_scope((void *)(drhd + 1),
210 ((void *)drhd) + drhd->header.length,
211 &dmaru->devices_cnt, &dmaru->devices,
214 list_del(&dmaru->list);
220 #ifdef CONFIG_ACPI_NUMA
222 dmar_parse_one_rhsa(struct acpi_dmar_header *header)
224 struct acpi_dmar_rhsa *rhsa;
225 struct dmar_drhd_unit *drhd;
227 rhsa = (struct acpi_dmar_rhsa *)header;
228 for_each_drhd_unit(drhd) {
229 if (drhd->reg_base_addr == rhsa->base_address) {
230 int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
232 if (!node_online(node))
234 drhd->iommu->node = node;
239 1, TAINT_FIRMWARE_WORKAROUND,
240 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
241 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
243 dmi_get_system_info(DMI_BIOS_VENDOR),
244 dmi_get_system_info(DMI_BIOS_VERSION),
245 dmi_get_system_info(DMI_PRODUCT_VERSION));
252 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
254 struct acpi_dmar_hardware_unit *drhd;
255 struct acpi_dmar_reserved_memory *rmrr;
256 struct acpi_dmar_atsr *atsr;
257 struct acpi_dmar_rhsa *rhsa;
259 switch (header->type) {
260 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
261 drhd = container_of(header, struct acpi_dmar_hardware_unit,
263 printk (KERN_INFO PREFIX
264 "DRHD base: %#016Lx flags: %#x\n",
265 (unsigned long long)drhd->address, drhd->flags);
267 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
268 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
270 printk (KERN_INFO PREFIX
271 "RMRR base: %#016Lx end: %#016Lx\n",
272 (unsigned long long)rmrr->base_address,
273 (unsigned long long)rmrr->end_address);
275 case ACPI_DMAR_TYPE_ATSR:
276 atsr = container_of(header, struct acpi_dmar_atsr, header);
277 printk(KERN_INFO PREFIX "ATSR flags: %#x\n", atsr->flags);
279 case ACPI_DMAR_HARDWARE_AFFINITY:
280 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
281 printk(KERN_INFO PREFIX "RHSA base: %#016Lx proximity domain: %#x\n",
282 (unsigned long long)rhsa->base_address,
283 rhsa->proximity_domain);
289 * dmar_table_detect - checks to see if the platform supports DMAR devices
291 static int __init dmar_table_detect(void)
293 acpi_status status = AE_OK;
295 /* if we could find DMAR table, then there are DMAR devices */
296 status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
297 (struct acpi_table_header **)&dmar_tbl,
300 if (ACPI_SUCCESS(status) && !dmar_tbl) {
301 printk (KERN_WARNING PREFIX "Unable to map DMAR\n");
302 status = AE_NOT_FOUND;
305 return (ACPI_SUCCESS(status) ? 1 : 0);
309 * parse_dmar_table - parses the DMA reporting table
312 parse_dmar_table(void)
314 struct acpi_table_dmar *dmar;
315 struct acpi_dmar_header *entry_header;
319 * Do it again, earlier dmar_tbl mapping could be mapped with
325 * ACPI tables may not be DMA protected by tboot, so use DMAR copy
326 * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
328 dmar_tbl = tboot_get_dmar_table(dmar_tbl);
330 dmar = (struct acpi_table_dmar *)dmar_tbl;
334 if (dmar->width < PAGE_SHIFT - 1) {
335 printk(KERN_WARNING PREFIX "Invalid DMAR haw\n");
339 printk (KERN_INFO PREFIX "Host address width %d\n",
342 entry_header = (struct acpi_dmar_header *)(dmar + 1);
343 while (((unsigned long)entry_header) <
344 (((unsigned long)dmar) + dmar_tbl->length)) {
345 /* Avoid looping forever on bad ACPI tables */
346 if (entry_header->length == 0) {
347 printk(KERN_WARNING PREFIX
348 "Invalid 0-length structure\n");
353 dmar_table_print_dmar_entry(entry_header);
355 switch (entry_header->type) {
356 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
357 ret = dmar_parse_one_drhd(entry_header);
359 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
360 ret = dmar_parse_one_rmrr(entry_header);
362 case ACPI_DMAR_TYPE_ATSR:
363 ret = dmar_parse_one_atsr(entry_header);
365 case ACPI_DMAR_HARDWARE_AFFINITY:
366 #ifdef CONFIG_ACPI_NUMA
367 ret = dmar_parse_one_rhsa(entry_header);
371 printk(KERN_WARNING PREFIX
372 "Unknown DMAR structure type %d\n",
374 ret = 0; /* for forward compatibility */
380 entry_header = ((void *)entry_header + entry_header->length);
385 static int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
391 for (index = 0; index < cnt; index++)
392 if (dev == devices[index])
395 /* Check our parent */
396 dev = dev->bus->self;
402 struct dmar_drhd_unit *
403 dmar_find_matched_drhd_unit(struct pci_dev *dev)
405 struct dmar_drhd_unit *dmaru = NULL;
406 struct acpi_dmar_hardware_unit *drhd;
408 dev = pci_physfn(dev);
410 list_for_each_entry(dmaru, &dmar_drhd_units, list) {
411 drhd = container_of(dmaru->hdr,
412 struct acpi_dmar_hardware_unit,
415 if (dmaru->include_all &&
416 drhd->segment == pci_domain_nr(dev->bus))
419 if (dmar_pci_device_match(dmaru->devices,
420 dmaru->devices_cnt, dev))
427 int __init dmar_dev_scope_init(void)
429 static int dmar_dev_scope_initialized;
430 struct dmar_drhd_unit *drhd, *drhd_n;
433 if (dmar_dev_scope_initialized)
434 return dmar_dev_scope_initialized;
436 if (list_empty(&dmar_drhd_units))
439 list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
440 ret = dmar_parse_dev(drhd);
445 ret = dmar_parse_rmrr_atsr_dev();
449 dmar_dev_scope_initialized = 1;
453 dmar_dev_scope_initialized = ret;
458 int __init dmar_table_init(void)
460 static int dmar_table_initialized;
463 if (dmar_table_initialized)
466 dmar_table_initialized = 1;
468 ret = parse_dmar_table();
471 printk(KERN_INFO PREFIX "parse DMAR table failure.\n");
475 if (list_empty(&dmar_drhd_units)) {
476 printk(KERN_INFO PREFIX "No DMAR devices found\n");
483 static void warn_invalid_dmar(u64 addr, const char *message)
486 1, TAINT_FIRMWARE_WORKAROUND,
487 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
488 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
490 dmi_get_system_info(DMI_BIOS_VENDOR),
491 dmi_get_system_info(DMI_BIOS_VERSION),
492 dmi_get_system_info(DMI_PRODUCT_VERSION));
495 int __init check_zero_address(void)
497 struct acpi_table_dmar *dmar;
498 struct acpi_dmar_header *entry_header;
499 struct acpi_dmar_hardware_unit *drhd;
501 dmar = (struct acpi_table_dmar *)dmar_tbl;
502 entry_header = (struct acpi_dmar_header *)(dmar + 1);
504 while (((unsigned long)entry_header) <
505 (((unsigned long)dmar) + dmar_tbl->length)) {
506 /* Avoid looping forever on bad ACPI tables */
507 if (entry_header->length == 0) {
508 printk(KERN_WARNING PREFIX
509 "Invalid 0-length structure\n");
513 if (entry_header->type == ACPI_DMAR_TYPE_HARDWARE_UNIT) {
517 drhd = (void *)entry_header;
518 if (!drhd->address) {
519 warn_invalid_dmar(0, "");
523 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
525 printk("IOMMU: can't validate: %llx\n", drhd->address);
528 cap = dmar_readq(addr + DMAR_CAP_REG);
529 ecap = dmar_readq(addr + DMAR_ECAP_REG);
530 early_iounmap(addr, VTD_PAGE_SIZE);
531 if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
532 warn_invalid_dmar(drhd->address,
533 " returns all ones");
538 entry_header = ((void *)entry_header + entry_header->length);
546 int __init detect_intel_iommu(void)
550 ret = dmar_table_detect();
552 ret = check_zero_address();
554 struct acpi_table_dmar *dmar;
556 dmar = (struct acpi_table_dmar *) dmar_tbl;
558 if (ret && intr_remapping_enabled && cpu_has_x2apic &&
561 "Queued invalidation will be enabled to support x2apic and Intr-remapping.\n");
563 if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
565 /* Make sure ACS will be enabled */
571 x86_init.iommu.iommu_init = intel_iommu_init;
574 early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
577 return ret ? 1 : -ENODEV;
581 int alloc_iommu(struct dmar_drhd_unit *drhd)
583 struct intel_iommu *iommu;
586 static int iommu_allocated = 0;
590 if (!drhd->reg_base_addr) {
591 warn_invalid_dmar(0, "");
595 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
599 iommu->seq_id = iommu_allocated++;
600 sprintf (iommu->name, "dmar%d", iommu->seq_id);
602 iommu->reg = ioremap(drhd->reg_base_addr, VTD_PAGE_SIZE);
604 printk(KERN_ERR "IOMMU: can't map the region\n");
607 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
608 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
610 if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
611 warn_invalid_dmar(drhd->reg_base_addr, " returns all ones");
615 agaw = iommu_calculate_agaw(iommu);
618 "Cannot get a valid agaw for iommu (seq_id = %d)\n",
622 msagaw = iommu_calculate_max_sagaw(iommu);
625 "Cannot get a valid max agaw for iommu (seq_id = %d)\n",
630 iommu->msagaw = msagaw;
634 /* the registers might be more than one page */
635 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
636 cap_max_fault_reg_offset(iommu->cap));
637 map_size = VTD_PAGE_ALIGN(map_size);
638 if (map_size > VTD_PAGE_SIZE) {
640 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
642 printk(KERN_ERR "IOMMU: can't map the region\n");
647 ver = readl(iommu->reg + DMAR_VER_REG);
648 pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
650 (unsigned long long)drhd->reg_base_addr,
651 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
652 (unsigned long long)iommu->cap,
653 (unsigned long long)iommu->ecap);
655 /* Reflect status in gcmd */
656 sts = readl(iommu->reg + DMAR_GSTS_REG);
657 if (sts & DMA_GSTS_IRES)
658 iommu->gcmd |= DMA_GCMD_IRE;
659 if (sts & DMA_GSTS_TES)
660 iommu->gcmd |= DMA_GCMD_TE;
661 if (sts & DMA_GSTS_QIES)
662 iommu->gcmd |= DMA_GCMD_QIE;
664 raw_spin_lock_init(&iommu->register_lock);
676 void free_iommu(struct intel_iommu *iommu)
681 free_dmar_iommu(iommu);
689 * Reclaim all the submitted descriptors which have completed its work.
691 static inline void reclaim_free_desc(struct q_inval *qi)
693 while (qi->desc_status[qi->free_tail] == QI_DONE ||
694 qi->desc_status[qi->free_tail] == QI_ABORT) {
695 qi->desc_status[qi->free_tail] = QI_FREE;
696 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
701 static int qi_check_fault(struct intel_iommu *iommu, int index)
705 struct q_inval *qi = iommu->qi;
706 int wait_index = (index + 1) % QI_LENGTH;
708 if (qi->desc_status[wait_index] == QI_ABORT)
711 fault = readl(iommu->reg + DMAR_FSTS_REG);
714 * If IQE happens, the head points to the descriptor associated
715 * with the error. No new descriptors are fetched until the IQE
718 if (fault & DMA_FSTS_IQE) {
719 head = readl(iommu->reg + DMAR_IQH_REG);
720 if ((head >> DMAR_IQ_SHIFT) == index) {
721 printk(KERN_ERR "VT-d detected invalid descriptor: "
722 "low=%llx, high=%llx\n",
723 (unsigned long long)qi->desc[index].low,
724 (unsigned long long)qi->desc[index].high);
725 memcpy(&qi->desc[index], &qi->desc[wait_index],
726 sizeof(struct qi_desc));
727 __iommu_flush_cache(iommu, &qi->desc[index],
728 sizeof(struct qi_desc));
729 writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
735 * If ITE happens, all pending wait_desc commands are aborted.
736 * No new descriptors are fetched until the ITE is cleared.
738 if (fault & DMA_FSTS_ITE) {
739 head = readl(iommu->reg + DMAR_IQH_REG);
740 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
742 tail = readl(iommu->reg + DMAR_IQT_REG);
743 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
745 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
748 if (qi->desc_status[head] == QI_IN_USE)
749 qi->desc_status[head] = QI_ABORT;
750 head = (head - 2 + QI_LENGTH) % QI_LENGTH;
751 } while (head != tail);
753 if (qi->desc_status[wait_index] == QI_ABORT)
757 if (fault & DMA_FSTS_ICE)
758 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
764 * Submit the queued invalidation descriptor to the remapping
765 * hardware unit and wait for its completion.
767 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
770 struct q_inval *qi = iommu->qi;
771 struct qi_desc *hw, wait_desc;
772 int wait_index, index;
783 raw_spin_lock_irqsave(&qi->q_lock, flags);
784 while (qi->free_cnt < 3) {
785 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
787 raw_spin_lock_irqsave(&qi->q_lock, flags);
790 index = qi->free_head;
791 wait_index = (index + 1) % QI_LENGTH;
793 qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
797 wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
798 QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
799 wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
801 hw[wait_index] = wait_desc;
803 __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
804 __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
806 qi->free_head = (qi->free_head + 2) % QI_LENGTH;
810 * update the HW tail register indicating the presence of
813 writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
815 while (qi->desc_status[wait_index] != QI_DONE) {
817 * We will leave the interrupts disabled, to prevent interrupt
818 * context to queue another cmd while a cmd is already submitted
819 * and waiting for completion on this cpu. This is to avoid
820 * a deadlock where the interrupt context can wait indefinitely
821 * for free slots in the queue.
823 rc = qi_check_fault(iommu, index);
827 raw_spin_unlock(&qi->q_lock);
829 raw_spin_lock(&qi->q_lock);
832 qi->desc_status[index] = QI_DONE;
834 reclaim_free_desc(qi);
835 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
844 * Flush the global interrupt entry cache.
846 void qi_global_iec(struct intel_iommu *iommu)
850 desc.low = QI_IEC_TYPE;
853 /* should never fail */
854 qi_submit_sync(&desc, iommu);
857 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
862 desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
863 | QI_CC_GRAN(type) | QI_CC_TYPE;
866 qi_submit_sync(&desc, iommu);
869 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
870 unsigned int size_order, u64 type)
877 if (cap_write_drain(iommu->cap))
880 if (cap_read_drain(iommu->cap))
883 desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
884 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
885 desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
886 | QI_IOTLB_AM(size_order);
888 qi_submit_sync(&desc, iommu);
891 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
892 u64 addr, unsigned mask)
897 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
898 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
899 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
901 desc.high = QI_DEV_IOTLB_ADDR(addr);
903 if (qdep >= QI_DEV_IOTLB_MAX_INVS)
906 desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
909 qi_submit_sync(&desc, iommu);
913 * Disable Queued Invalidation interface.
915 void dmar_disable_qi(struct intel_iommu *iommu)
919 cycles_t start_time = get_cycles();
921 if (!ecap_qis(iommu->ecap))
924 raw_spin_lock_irqsave(&iommu->register_lock, flags);
926 sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
927 if (!(sts & DMA_GSTS_QIES))
931 * Give a chance to HW to complete the pending invalidation requests.
933 while ((readl(iommu->reg + DMAR_IQT_REG) !=
934 readl(iommu->reg + DMAR_IQH_REG)) &&
935 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
938 iommu->gcmd &= ~DMA_GCMD_QIE;
939 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
941 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
942 !(sts & DMA_GSTS_QIES), sts);
944 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
948 * Enable queued invalidation.
950 static void __dmar_enable_qi(struct intel_iommu *iommu)
954 struct q_inval *qi = iommu->qi;
956 qi->free_head = qi->free_tail = 0;
957 qi->free_cnt = QI_LENGTH;
959 raw_spin_lock_irqsave(&iommu->register_lock, flags);
961 /* write zero to the tail reg */
962 writel(0, iommu->reg + DMAR_IQT_REG);
964 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
966 iommu->gcmd |= DMA_GCMD_QIE;
967 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
969 /* Make sure hardware complete it */
970 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
972 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
976 * Enable Queued Invalidation interface. This is a must to support
977 * interrupt-remapping. Also used by DMA-remapping, which replaces
978 * register based IOTLB invalidation.
980 int dmar_enable_qi(struct intel_iommu *iommu)
983 struct page *desc_page;
985 if (!ecap_qis(iommu->ecap))
989 * queued invalidation is already setup and enabled.
994 iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1001 desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1008 qi->desc = page_address(desc_page);
1010 qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1011 if (!qi->desc_status) {
1012 free_page((unsigned long) qi->desc);
1018 qi->free_head = qi->free_tail = 0;
1019 qi->free_cnt = QI_LENGTH;
1021 raw_spin_lock_init(&qi->q_lock);
1023 __dmar_enable_qi(iommu);
1028 /* iommu interrupt handling. Most stuff are MSI-like. */
1036 static const char *dma_remap_fault_reasons[] =
1039 "Present bit in root entry is clear",
1040 "Present bit in context entry is clear",
1041 "Invalid context entry",
1042 "Access beyond MGAW",
1043 "PTE Write access is not set",
1044 "PTE Read access is not set",
1045 "Next page table ptr is invalid",
1046 "Root table address invalid",
1047 "Context table ptr is invalid",
1048 "non-zero reserved fields in RTP",
1049 "non-zero reserved fields in CTP",
1050 "non-zero reserved fields in PTE",
1053 static const char *intr_remap_fault_reasons[] =
1055 "Detected reserved fields in the decoded interrupt-remapped request",
1056 "Interrupt index exceeded the interrupt-remapping table size",
1057 "Present field in the IRTE entry is clear",
1058 "Error accessing interrupt-remapping table pointed by IRTA_REG",
1059 "Detected reserved fields in the IRTE entry",
1060 "Blocked a compatibility format interrupt request",
1061 "Blocked an interrupt request due to source-id verification failure",
1064 #define MAX_FAULT_REASON_IDX (ARRAY_SIZE(fault_reason_strings) - 1)
1066 const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1068 if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1069 ARRAY_SIZE(intr_remap_fault_reasons))) {
1070 *fault_type = INTR_REMAP;
1071 return intr_remap_fault_reasons[fault_reason - 0x20];
1072 } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1073 *fault_type = DMA_REMAP;
1074 return dma_remap_fault_reasons[fault_reason];
1076 *fault_type = UNKNOWN;
1081 void dmar_msi_unmask(struct irq_data *data)
1083 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1087 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1088 writel(0, iommu->reg + DMAR_FECTL_REG);
1089 /* Read a reg to force flush the post write */
1090 readl(iommu->reg + DMAR_FECTL_REG);
1091 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1094 void dmar_msi_mask(struct irq_data *data)
1097 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1100 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1101 writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1102 /* Read a reg to force flush the post write */
1103 readl(iommu->reg + DMAR_FECTL_REG);
1104 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1107 void dmar_msi_write(int irq, struct msi_msg *msg)
1109 struct intel_iommu *iommu = irq_get_handler_data(irq);
1112 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1113 writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1114 writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1115 writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1116 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1119 void dmar_msi_read(int irq, struct msi_msg *msg)
1121 struct intel_iommu *iommu = irq_get_handler_data(irq);
1124 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1125 msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1126 msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1127 msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1128 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1131 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1132 u8 fault_reason, u16 source_id, unsigned long long addr)
1137 reason = dmar_get_fault_reason(fault_reason, &fault_type);
1139 if (fault_type == INTR_REMAP)
1140 printk(KERN_ERR "INTR-REMAP: Request device [[%02x:%02x.%d] "
1141 "fault index %llx\n"
1142 "INTR-REMAP:[fault reason %02d] %s\n",
1143 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1144 PCI_FUNC(source_id & 0xFF), addr >> 48,
1145 fault_reason, reason);
1148 "DMAR:[%s] Request device [%02x:%02x.%d] "
1149 "fault addr %llx \n"
1150 "DMAR:[fault reason %02d] %s\n",
1151 (type ? "DMA Read" : "DMA Write"),
1152 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1153 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1157 #define PRIMARY_FAULT_REG_LEN (16)
1158 irqreturn_t dmar_fault(int irq, void *dev_id)
1160 struct intel_iommu *iommu = dev_id;
1161 int reg, fault_index;
1165 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1166 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1168 printk(KERN_ERR "DRHD: handling fault status reg %x\n",
1171 /* TBD: ignore advanced fault log currently */
1172 if (!(fault_status & DMA_FSTS_PPF))
1175 fault_index = dma_fsts_fault_record_index(fault_status);
1176 reg = cap_fault_reg_offset(iommu->cap);
1184 /* highest 32 bits */
1185 data = readl(iommu->reg + reg +
1186 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1187 if (!(data & DMA_FRCD_F))
1190 fault_reason = dma_frcd_fault_reason(data);
1191 type = dma_frcd_type(data);
1193 data = readl(iommu->reg + reg +
1194 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1195 source_id = dma_frcd_source_id(data);
1197 guest_addr = dmar_readq(iommu->reg + reg +
1198 fault_index * PRIMARY_FAULT_REG_LEN);
1199 guest_addr = dma_frcd_page_addr(guest_addr);
1200 /* clear the fault */
1201 writel(DMA_FRCD_F, iommu->reg + reg +
1202 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1204 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1206 dmar_fault_do_one(iommu, type, fault_reason,
1207 source_id, guest_addr);
1210 if (fault_index >= cap_num_fault_regs(iommu->cap))
1212 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1215 /* clear all the other faults */
1216 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1217 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1219 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1223 int dmar_set_interrupt(struct intel_iommu *iommu)
1228 * Check if the fault interrupt is already initialized.
1235 printk(KERN_ERR "IOMMU: no free vectors\n");
1239 irq_set_handler_data(irq, iommu);
1242 ret = arch_setup_dmar_msi(irq);
1244 irq_set_handler_data(irq, NULL);
1250 ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1252 printk(KERN_ERR "IOMMU: can't request irq\n");
1256 int __init enable_drhd_fault_handling(void)
1258 struct dmar_drhd_unit *drhd;
1261 * Enable fault control interrupt.
1263 for_each_drhd_unit(drhd) {
1265 struct intel_iommu *iommu = drhd->iommu;
1266 ret = dmar_set_interrupt(iommu);
1269 printk(KERN_ERR "DRHD %Lx: failed to enable fault, "
1270 " interrupt, ret %d\n",
1271 (unsigned long long)drhd->reg_base_addr, ret);
1276 * Clear any previous faults.
1278 dmar_fault(iommu->irq, iommu);
1285 * Re-enable Queued Invalidation interface.
1287 int dmar_reenable_qi(struct intel_iommu *iommu)
1289 if (!ecap_qis(iommu->ecap))
1296 * First disable queued invalidation.
1298 dmar_disable_qi(iommu);
1300 * Then enable queued invalidation again. Since there is no pending
1301 * invalidation requests now, it's safe to re-enable queued
1304 __dmar_enable_qi(iommu);
1310 * Check interrupt remapping support in DMAR table description.
1312 int __init dmar_ir_support(void)
1314 struct acpi_table_dmar *dmar;
1315 dmar = (struct acpi_table_dmar *)dmar_tbl;
1318 return dmar->flags & 0x1;
1320 IOMMU_INIT_POST(detect_intel_iommu);