1 #include <linux/dma-mapping.h>
2 #include <linux/dmar.h>
3 #include <linux/bootmem.h>
9 #include <asm/calgary.h>
10 #include <asm/amd_iommu.h>
12 int forbid_dac __read_mostly;
13 EXPORT_SYMBOL(forbid_dac);
15 const struct dma_mapping_ops *dma_ops;
16 EXPORT_SYMBOL(dma_ops);
18 static int iommu_sac_force __read_mostly;
20 #ifdef CONFIG_IOMMU_DEBUG
21 int panic_on_overflow __read_mostly = 1;
22 int force_iommu __read_mostly = 1;
24 int panic_on_overflow __read_mostly = 0;
25 int force_iommu __read_mostly = 0;
28 int iommu_merge __read_mostly = 0;
30 int no_iommu __read_mostly;
31 /* Set this to 1 if there is a HW IOMMU in the system */
32 int iommu_detected __read_mostly = 0;
34 /* This tells the BIO block layer to assume merging. Default to off
35 because we cannot guarantee merging later. */
36 int iommu_bio_merge __read_mostly = 0;
37 EXPORT_SYMBOL(iommu_bio_merge);
39 dma_addr_t bad_dma_address __read_mostly = 0;
40 EXPORT_SYMBOL(bad_dma_address);
42 /* Dummy device used for NULL arguments (normally ISA). Better would
43 be probably a smaller DMA mask, but this is bug-to-bug compatible
45 struct device fallback_dev = {
46 .bus_id = "fallback device",
47 .coherent_dma_mask = DMA_32BIT_MASK,
48 .dma_mask = &fallback_dev.coherent_dma_mask,
51 int dma_set_mask(struct device *dev, u64 mask)
53 if (!dev->dma_mask || !dma_supported(dev, mask))
56 *dev->dma_mask = mask;
60 EXPORT_SYMBOL(dma_set_mask);
63 static __initdata void *dma32_bootmem_ptr;
64 static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);
66 static int __init parse_dma32_size_opt(char *p)
70 dma32_bootmem_size = memparse(p, &p);
73 early_param("dma32_size", parse_dma32_size_opt);
75 void __init dma32_reserve_bootmem(void)
77 unsigned long size, align;
78 if (max_pfn <= MAX_DMA32_PFN)
82 * check aperture_64.c allocate_aperture() for reason about
86 size = round_up(dma32_bootmem_size, align);
87 dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
89 if (dma32_bootmem_ptr)
90 dma32_bootmem_size = size;
92 dma32_bootmem_size = 0;
94 static void __init dma32_free_bootmem(void)
97 if (max_pfn <= MAX_DMA32_PFN)
100 if (!dma32_bootmem_ptr)
103 free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size);
105 dma32_bootmem_ptr = NULL;
106 dma32_bootmem_size = 0;
109 void __init pci_iommu_alloc(void)
111 /* free the range so iommu could get some range less than 4G */
112 dma32_free_bootmem();
114 * The order of these functions is important for
115 * fall-back/fail-over reasons
117 #ifdef CONFIG_GART_IOMMU
118 gart_iommu_hole_init();
121 #ifdef CONFIG_CALGARY_IOMMU
125 detect_intel_iommu();
129 #ifdef CONFIG_SWIOTLB
136 * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
139 static __init int iommu_setup(char *p)
147 if (!strncmp(p, "off", 3))
149 /* gart_parse_options has more force support */
150 if (!strncmp(p, "force", 5))
152 if (!strncmp(p, "noforce", 7)) {
157 if (!strncmp(p, "biomerge", 8)) {
158 iommu_bio_merge = 4096;
162 if (!strncmp(p, "panic", 5))
163 panic_on_overflow = 1;
164 if (!strncmp(p, "nopanic", 7))
165 panic_on_overflow = 0;
166 if (!strncmp(p, "merge", 5)) {
170 if (!strncmp(p, "nomerge", 7))
172 if (!strncmp(p, "forcesac", 8))
174 if (!strncmp(p, "allowdac", 8))
176 if (!strncmp(p, "nodac", 5))
178 if (!strncmp(p, "usedac", 6)) {
182 #ifdef CONFIG_SWIOTLB
183 if (!strncmp(p, "soft", 4))
187 #ifdef CONFIG_GART_IOMMU
188 gart_parse_options(p);
191 #ifdef CONFIG_CALGARY_IOMMU
192 if (!strncmp(p, "calgary", 7))
194 #endif /* CONFIG_CALGARY_IOMMU */
196 p += strcspn(p, ",");
202 early_param("iommu", iommu_setup);
204 int dma_supported(struct device *dev, u64 mask)
207 if (mask > 0xffffffff && forbid_dac > 0) {
208 printk(KERN_INFO "PCI: Disallowing DAC for device %s\n",
214 if (dma_ops->dma_supported)
215 return dma_ops->dma_supported(dev, mask);
217 /* Copied from i386. Doesn't make much sense, because it will
218 only work for pci_alloc_coherent.
219 The caller just has to use GFP_DMA in this case. */
220 if (mask < DMA_24BIT_MASK)
223 /* Tell the device to use SAC when IOMMU force is on. This
224 allows the driver to use cheaper accesses in some cases.
226 Problem with this is that if we overflow the IOMMU area and
227 return DAC as fallback address the device may not handle it
230 As a special case some controllers have a 39bit address
231 mode that is as efficient as 32bit (aic79xx). Don't force
232 SAC for these. Assume all masks <= 40 bits are of this
233 type. Normally this doesn't make any difference, but gives
234 more gentle handling of IOMMU overflow. */
235 if (iommu_sac_force && (mask >= DMA_40BIT_MASK)) {
236 printk(KERN_INFO "%s: Force SAC with mask %Lx\n",
243 EXPORT_SYMBOL(dma_supported);
245 /* Allocate DMA memory on node near device */
246 static noinline struct page *
247 dma_alloc_pages(struct device *dev, gfp_t gfp, unsigned order)
251 node = dev_to_node(dev);
253 return alloc_pages_node(node, gfp, order);
257 * Allocate memory for a coherent mapping.
260 dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
265 unsigned long dma_mask = 0;
269 /* ignore region specifiers */
270 gfp &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
272 if (dma_alloc_from_coherent(dev, size, dma_handle, &memory))
279 dma_mask = dev->coherent_dma_mask;
281 dma_mask = (gfp & GFP_DMA) ? DMA_24BIT_MASK : DMA_32BIT_MASK;
283 /* Device not DMA able */
284 if (dev->dma_mask == NULL)
287 /* Don't invoke OOM killer or retry in lower 16MB DMA zone */
292 /* Why <=? Even when the mask is smaller than 4GB it is often
293 larger than 16MB and in this case we have a chance of
294 finding fitting memory in the next higher zone first. If
295 not retry with true GFP_DMA. -AK */
296 if (dma_mask <= DMA_32BIT_MASK && !(gfp & GFP_DMA)) {
298 if (dma_mask < DMA_32BIT_MASK)
304 page = dma_alloc_pages(dev,
305 noretry ? gfp | __GFP_NORETRY : gfp, get_order(size));
311 bus = page_to_phys(page);
312 memory = page_address(page);
313 high = (bus + size) >= dma_mask;
315 if (force_iommu && !(gfp & GFP_DMA))
318 free_pages((unsigned long)memory,
321 /* Don't use the 16MB ZONE_DMA unless absolutely
322 needed. It's better to use remapping first. */
323 if (dma_mask < DMA_32BIT_MASK && !(gfp & GFP_DMA)) {
324 gfp = (gfp & ~GFP_DMA32) | GFP_DMA;
328 /* Let low level make its own zone decisions */
329 gfp &= ~(GFP_DMA32|GFP_DMA);
331 if (dma_ops->alloc_coherent)
332 return dma_ops->alloc_coherent(dev, size,
337 memset(memory, 0, size);
344 if (dma_ops->alloc_coherent) {
345 free_pages((unsigned long)memory, get_order(size));
346 gfp &= ~(GFP_DMA|GFP_DMA32);
347 return dma_ops->alloc_coherent(dev, size, dma_handle, gfp);
350 if (dma_ops->map_simple) {
351 *dma_handle = dma_ops->map_simple(dev, virt_to_phys(memory),
353 PCI_DMA_BIDIRECTIONAL);
354 if (*dma_handle != bad_dma_address)
358 if (panic_on_overflow)
359 panic("dma_alloc_coherent: IOMMU overflow by %lu bytes\n",
360 (unsigned long)size);
361 free_pages((unsigned long)memory, get_order(size));
364 EXPORT_SYMBOL(dma_alloc_coherent);
367 * Unmap coherent memory.
368 * The caller must ensure that the device has finished accessing the mapping.
370 void dma_free_coherent(struct device *dev, size_t size,
371 void *vaddr, dma_addr_t bus)
373 int order = get_order(size);
374 WARN_ON(irqs_disabled()); /* for portability */
375 if (dma_release_from_coherent(dev, order, vaddr))
377 if (dma_ops->unmap_single)
378 dma_ops->unmap_single(dev, bus, size, 0);
379 free_pages((unsigned long)vaddr, order);
381 EXPORT_SYMBOL(dma_free_coherent);
383 static int __init pci_iommu_init(void)
385 #ifdef CONFIG_CALGARY_IOMMU
386 calgary_iommu_init();
393 #ifdef CONFIG_GART_IOMMU
401 void pci_iommu_shutdown(void)
403 gart_iommu_shutdown();
405 /* Must execute after PCI subsystem */
406 fs_initcall(pci_iommu_init);
409 /* Many VIA bridges seem to corrupt data for DAC. Disable it here */
411 static __devinit void via_no_dac(struct pci_dev *dev)
413 if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
414 printk(KERN_INFO "PCI: VIA PCI bridge detected."
419 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);