2 * linux/arch/arm/mm/dma-mapping.c
4 * Copyright (C) 2000-2004 Russell King
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 * DMA uncached mapping support.
12 #include <linux/module.h>
14 #include <linux/gfp.h>
15 #include <linux/errno.h>
16 #include <linux/list.h>
17 #include <linux/init.h>
18 #include <linux/device.h>
19 #include <linux/dma-mapping.h>
20 #include <linux/highmem.h>
21 #include <linux/slab.h>
23 #include <asm/memory.h>
24 #include <asm/highmem.h>
25 #include <asm/cacheflush.h>
26 #include <asm/tlbflush.h>
27 #include <asm/sizes.h>
28 #include <asm/mach/arch.h>
33 * The DMA API is built upon the notion of "buffer ownership". A buffer
34 * is either exclusively owned by the CPU (and therefore may be accessed
35 * by it) or exclusively owned by the DMA device. These helper functions
36 * represent the transitions between these two ownership states.
38 * Note, however, that on later ARMs, this notion does not work due to
39 * speculative prefetches. We model our approach on the assumption that
40 * the CPU does do speculative prefetches, which means we clean caches
41 * before transfers and delay cache invalidation until transfer completion.
43 * Private support functions: these are not part of the API and are
44 * liable to change. Drivers must not use these.
46 static inline void __dma_single_cpu_to_dev(const void *kaddr, size_t size,
47 enum dma_data_direction dir)
49 extern void ___dma_single_cpu_to_dev(const void *, size_t,
50 enum dma_data_direction);
52 if (!arch_is_coherent())
53 ___dma_single_cpu_to_dev(kaddr, size, dir);
56 static inline void __dma_single_dev_to_cpu(const void *kaddr, size_t size,
57 enum dma_data_direction dir)
59 extern void ___dma_single_dev_to_cpu(const void *, size_t,
60 enum dma_data_direction);
62 if (!arch_is_coherent())
63 ___dma_single_dev_to_cpu(kaddr, size, dir);
66 static inline void __dma_page_cpu_to_dev(struct page *page, unsigned long off,
67 size_t size, enum dma_data_direction dir)
69 extern void ___dma_page_cpu_to_dev(struct page *, unsigned long,
70 size_t, enum dma_data_direction);
72 if (!arch_is_coherent())
73 ___dma_page_cpu_to_dev(page, off, size, dir);
76 static inline void __dma_page_dev_to_cpu(struct page *page, unsigned long off,
77 size_t size, enum dma_data_direction dir)
79 extern void ___dma_page_dev_to_cpu(struct page *, unsigned long,
80 size_t, enum dma_data_direction);
82 if (!arch_is_coherent())
83 ___dma_page_dev_to_cpu(page, off, size, dir);
87 static inline dma_addr_t __dma_map_page(struct device *dev, struct page *page,
88 unsigned long offset, size_t size, enum dma_data_direction dir)
90 __dma_page_cpu_to_dev(page, offset, size, dir);
91 return pfn_to_dma(dev, page_to_pfn(page)) + offset;
94 static inline void __dma_unmap_page(struct device *dev, dma_addr_t handle,
95 size_t size, enum dma_data_direction dir)
97 __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)),
98 handle & ~PAGE_MASK, size, dir);
102 * arm_dma_map_page - map a portion of a page for streaming DMA
103 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
104 * @page: page that buffer resides in
105 * @offset: offset into page for start of buffer
106 * @size: size of buffer to map
107 * @dir: DMA transfer direction
109 * Ensure that any data held in the cache is appropriately discarded
112 * The device owns this memory once this call has completed. The CPU
113 * can regain ownership by calling dma_unmap_page().
115 static inline dma_addr_t arm_dma_map_page(struct device *dev, struct page *page,
116 unsigned long offset, size_t size, enum dma_data_direction dir,
117 struct dma_attrs *attrs)
119 return __dma_map_page(dev, page, offset, size, dir);
123 * arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page()
124 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
125 * @handle: DMA address of buffer
126 * @size: size of buffer (same as passed to dma_map_page)
127 * @dir: DMA transfer direction (same as passed to dma_map_page)
129 * Unmap a page streaming mode DMA translation. The handle and size
130 * must match what was provided in the previous dma_map_page() call.
131 * All other usages are undefined.
133 * After this call, reads by the CPU to the buffer are guaranteed to see
134 * whatever the device wrote there.
136 static inline void arm_dma_unmap_page(struct device *dev, dma_addr_t handle,
137 size_t size, enum dma_data_direction dir,
138 struct dma_attrs *attrs)
140 __dma_unmap_page(dev, handle, size, dir);
143 static inline void arm_dma_sync_single_for_cpu(struct device *dev,
144 dma_addr_t handle, size_t size, enum dma_data_direction dir)
146 unsigned int offset = handle & (PAGE_SIZE - 1);
147 struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
148 __dma_page_dev_to_cpu(page, offset, size, dir);
151 static inline void arm_dma_sync_single_for_device(struct device *dev,
152 dma_addr_t handle, size_t size, enum dma_data_direction dir)
154 unsigned int offset = handle & (PAGE_SIZE - 1);
155 struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
156 __dma_page_cpu_to_dev(page, offset, size, dir);
159 static int arm_dma_set_mask(struct device *dev, u64 dma_mask);
161 struct dma_map_ops arm_dma_ops = {
162 .map_page = arm_dma_map_page,
163 .unmap_page = arm_dma_unmap_page,
164 .map_sg = arm_dma_map_sg,
165 .unmap_sg = arm_dma_unmap_sg,
166 .sync_single_for_cpu = arm_dma_sync_single_for_cpu,
167 .sync_single_for_device = arm_dma_sync_single_for_device,
168 .sync_sg_for_cpu = arm_dma_sync_sg_for_cpu,
169 .sync_sg_for_device = arm_dma_sync_sg_for_device,
170 .set_dma_mask = arm_dma_set_mask,
172 EXPORT_SYMBOL(arm_dma_ops);
174 static u64 get_coherent_dma_mask(struct device *dev)
176 u64 mask = (u64)arm_dma_limit;
179 mask = dev->coherent_dma_mask;
182 * Sanity check the DMA mask - it must be non-zero, and
183 * must be able to be satisfied by a DMA allocation.
186 dev_warn(dev, "coherent DMA mask is unset\n");
190 if ((~mask) & (u64)arm_dma_limit) {
191 dev_warn(dev, "coherent DMA mask %#llx is smaller "
192 "than system GFP_DMA mask %#llx\n",
193 mask, (u64)arm_dma_limit);
202 * Allocate a DMA buffer for 'dev' of size 'size' using the
203 * specified gfp mask. Note that 'size' must be page aligned.
205 static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp)
207 unsigned long order = get_order(size);
208 struct page *page, *p, *e;
210 u64 mask = get_coherent_dma_mask(dev);
212 #ifdef CONFIG_DMA_API_DEBUG
213 u64 limit = (mask + 1) & ~mask;
214 if (limit && size >= limit) {
215 dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n",
224 if (mask < 0xffffffffULL)
227 page = alloc_pages(gfp, order);
232 * Now split the huge page and free the excess pages
234 split_page(page, order);
235 for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++)
239 * Ensure that the allocated pages are zeroed, and that any data
240 * lurking in the kernel direct-mapped region is invalidated.
242 ptr = page_address(page);
243 memset(ptr, 0, size);
244 dmac_flush_range(ptr, ptr + size);
245 outer_flush_range(__pa(ptr), __pa(ptr) + size);
251 * Free a DMA buffer. 'size' must be page aligned.
253 static void __dma_free_buffer(struct page *page, size_t size)
255 struct page *e = page + (size >> PAGE_SHIFT);
265 #define CONSISTENT_OFFSET(x) (((unsigned long)(x) - consistent_base) >> PAGE_SHIFT)
266 #define CONSISTENT_PTE_INDEX(x) (((unsigned long)(x) - consistent_base) >> PMD_SHIFT)
269 * These are the page tables (2MB each) covering uncached, DMA consistent allocations
271 static pte_t **consistent_pte;
273 #define DEFAULT_CONSISTENT_DMA_SIZE SZ_2M
275 unsigned long consistent_base = CONSISTENT_END - DEFAULT_CONSISTENT_DMA_SIZE;
277 void __init init_consistent_dma_size(unsigned long size)
279 unsigned long base = CONSISTENT_END - ALIGN(size, SZ_2M);
281 BUG_ON(consistent_pte); /* Check we're called before DMA region init */
282 BUG_ON(base < VMALLOC_END);
284 /* Grow region to accommodate specified size */
285 if (base < consistent_base)
286 consistent_base = base;
289 #include "vmregion.h"
291 static struct arm_vmregion_head consistent_head = {
292 .vm_lock = __SPIN_LOCK_UNLOCKED(&consistent_head.vm_lock),
293 .vm_list = LIST_HEAD_INIT(consistent_head.vm_list),
294 .vm_end = CONSISTENT_END,
297 #ifdef CONFIG_HUGETLB_PAGE
298 #warning ARM Coherent DMA allocator does not (yet) support huge TLB
302 * Initialise the consistent memory allocation.
304 static int __init consistent_init(void)
312 unsigned long base = consistent_base;
313 unsigned long num_ptes = (CONSISTENT_END - base) >> PMD_SHIFT;
315 consistent_pte = kmalloc(num_ptes * sizeof(pte_t), GFP_KERNEL);
316 if (!consistent_pte) {
317 pr_err("%s: no memory\n", __func__);
321 pr_debug("DMA memory: 0x%08lx - 0x%08lx:\n", base, CONSISTENT_END);
322 consistent_head.vm_start = base;
325 pgd = pgd_offset(&init_mm, base);
327 pud = pud_alloc(&init_mm, pgd, base);
329 pr_err("%s: no pud tables\n", __func__);
334 pmd = pmd_alloc(&init_mm, pud, base);
336 pr_err("%s: no pmd tables\n", __func__);
340 WARN_ON(!pmd_none(*pmd));
342 pte = pte_alloc_kernel(pmd, base);
344 pr_err("%s: no pte tables\n", __func__);
349 consistent_pte[i++] = pte;
351 } while (base < CONSISTENT_END);
356 core_initcall(consistent_init);
359 __dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot,
362 struct arm_vmregion *c;
366 if (!consistent_pte) {
367 pr_err("%s: not initialised\n", __func__);
373 * Align the virtual region allocation - maximum alignment is
374 * a section size, minimum is a page size. This helps reduce
375 * fragmentation of the DMA space, and also prevents allocations
376 * smaller than a section from crossing a section boundary.
379 if (bit > SECTION_SHIFT)
384 * Allocate a virtual address in the consistent mapping region.
386 c = arm_vmregion_alloc(&consistent_head, align, size,
387 gfp & ~(__GFP_DMA | __GFP_HIGHMEM), caller);
390 int idx = CONSISTENT_PTE_INDEX(c->vm_start);
391 u32 off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1);
393 pte = consistent_pte[idx] + off;
397 BUG_ON(!pte_none(*pte));
399 set_pte_ext(pte, mk_pte(page, prot), 0);
403 if (off >= PTRS_PER_PTE) {
405 pte = consistent_pte[++idx];
407 } while (size -= PAGE_SIZE);
411 return (void *)c->vm_start;
416 static void __dma_free_remap(void *cpu_addr, size_t size)
418 struct arm_vmregion *c;
424 c = arm_vmregion_find_remove(&consistent_head, (unsigned long)cpu_addr);
426 pr_err("%s: trying to free invalid coherent area: %p\n",
432 if ((c->vm_end - c->vm_start) != size) {
433 pr_err("%s: freeing wrong coherent size (%ld != %d)\n",
434 __func__, c->vm_end - c->vm_start, size);
436 size = c->vm_end - c->vm_start;
439 idx = CONSISTENT_PTE_INDEX(c->vm_start);
440 off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1);
441 ptep = consistent_pte[idx] + off;
444 pte_t pte = ptep_get_and_clear(&init_mm, addr, ptep);
449 if (off >= PTRS_PER_PTE) {
451 ptep = consistent_pte[++idx];
454 if (pte_none(pte) || !pte_present(pte))
455 pr_crit("%s: bad page in kernel page table\n",
457 } while (size -= PAGE_SIZE);
459 flush_tlb_kernel_range(c->vm_start, c->vm_end);
461 arm_vmregion_free(&consistent_head, c);
464 #else /* !CONFIG_MMU */
466 #define __dma_alloc_remap(page, size, gfp, prot, c) page_address(page)
467 #define __dma_free_remap(addr, size) do { } while (0)
469 #endif /* CONFIG_MMU */
472 __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp,
473 pgprot_t prot, const void *caller)
479 * Following is a work-around (a.k.a. hack) to prevent pages
480 * with __GFP_COMP being passed to split_page() which cannot
481 * handle them. The real problem is that this flag probably
482 * should be 0 on ARM as it is not supported on this
483 * platform; see CONFIG_HUGETLBFS.
485 gfp &= ~(__GFP_COMP);
487 *handle = DMA_ERROR_CODE;
488 size = PAGE_ALIGN(size);
490 page = __dma_alloc_buffer(dev, size, gfp);
494 if (!arch_is_coherent())
495 addr = __dma_alloc_remap(page, size, gfp, prot, caller);
497 addr = page_address(page);
500 *handle = pfn_to_dma(dev, page_to_pfn(page));
502 __dma_free_buffer(page, size);
508 * Allocate DMA-coherent memory space and return both the kernel remapped
509 * virtual and bus address for that space.
512 dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp)
516 if (dma_alloc_from_coherent(dev, size, handle, &memory))
519 return __dma_alloc(dev, size, handle, gfp,
520 pgprot_dmacoherent(pgprot_kernel),
521 __builtin_return_address(0));
523 EXPORT_SYMBOL(dma_alloc_coherent);
526 * Allocate a writecombining region, in much the same way as
527 * dma_alloc_coherent above.
530 dma_alloc_writecombine(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp)
532 return __dma_alloc(dev, size, handle, gfp,
533 pgprot_writecombine(pgprot_kernel),
534 __builtin_return_address(0));
536 EXPORT_SYMBOL(dma_alloc_writecombine);
538 static int dma_mmap(struct device *dev, struct vm_area_struct *vma,
539 void *cpu_addr, dma_addr_t dma_addr, size_t size)
543 unsigned long user_size, kern_size;
544 struct arm_vmregion *c;
546 if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
549 user_size = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
551 c = arm_vmregion_find(&consistent_head, (unsigned long)cpu_addr);
553 unsigned long off = vma->vm_pgoff;
555 kern_size = (c->vm_end - c->vm_start) >> PAGE_SHIFT;
557 if (off < kern_size &&
558 user_size <= (kern_size - off)) {
559 ret = remap_pfn_range(vma, vma->vm_start,
560 page_to_pfn(c->vm_pages) + off,
561 user_size << PAGE_SHIFT,
565 #endif /* CONFIG_MMU */
570 int dma_mmap_coherent(struct device *dev, struct vm_area_struct *vma,
571 void *cpu_addr, dma_addr_t dma_addr, size_t size)
573 vma->vm_page_prot = pgprot_dmacoherent(vma->vm_page_prot);
574 return dma_mmap(dev, vma, cpu_addr, dma_addr, size);
576 EXPORT_SYMBOL(dma_mmap_coherent);
578 int dma_mmap_writecombine(struct device *dev, struct vm_area_struct *vma,
579 void *cpu_addr, dma_addr_t dma_addr, size_t size)
581 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
582 return dma_mmap(dev, vma, cpu_addr, dma_addr, size);
584 EXPORT_SYMBOL(dma_mmap_writecombine);
587 * free a page as defined by the above mapping.
588 * Must not be called with IRQs disabled.
590 void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr_t handle)
592 WARN_ON(irqs_disabled());
594 if (dma_release_from_coherent(dev, get_order(size), cpu_addr))
597 size = PAGE_ALIGN(size);
599 if (!arch_is_coherent())
600 __dma_free_remap(cpu_addr, size);
602 __dma_free_buffer(pfn_to_page(dma_to_pfn(dev, handle)), size);
604 EXPORT_SYMBOL(dma_free_coherent);
606 static void dma_cache_maint_page(struct page *page, unsigned long offset,
607 size_t size, enum dma_data_direction dir,
608 void (*op)(const void *, size_t, int))
613 pfn = page_to_pfn(page) + offset / PAGE_SIZE;
617 * A single sg entry may refer to multiple physically contiguous
618 * pages. But we still need to process highmem pages individually.
619 * If highmem is not configured then the bulk of this loop gets
626 page = pfn_to_page(pfn);
628 if (PageHighMem(page)) {
629 if (len + offset > PAGE_SIZE)
630 len = PAGE_SIZE - offset;
631 vaddr = kmap_high_get(page);
636 } else if (cache_is_vipt()) {
637 /* unmapped pages might still be cached */
638 vaddr = kmap_atomic(page);
639 op(vaddr + offset, len, dir);
640 kunmap_atomic(vaddr);
643 vaddr = page_address(page) + offset;
652 void ___dma_page_cpu_to_dev(struct page *page, unsigned long off,
653 size_t size, enum dma_data_direction dir)
657 dma_cache_maint_page(page, off, size, dir, dmac_map_area);
659 paddr = page_to_phys(page) + off;
660 if (dir == DMA_FROM_DEVICE) {
661 outer_inv_range(paddr, paddr + size);
663 outer_clean_range(paddr, paddr + size);
665 /* FIXME: non-speculating: flush on bidirectional mappings? */
668 void ___dma_page_dev_to_cpu(struct page *page, unsigned long off,
669 size_t size, enum dma_data_direction dir)
671 unsigned long paddr = page_to_phys(page) + off;
673 /* FIXME: non-speculating: not required */
674 /* don't bother invalidating if DMA to device */
675 if (dir != DMA_TO_DEVICE)
676 outer_inv_range(paddr, paddr + size);
678 dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);
681 * Mark the D-cache clean for this page to avoid extra flushing.
683 if (dir != DMA_TO_DEVICE && off == 0 && size >= PAGE_SIZE)
684 set_bit(PG_dcache_clean, &page->flags);
688 * arm_dma_map_sg - map a set of SG buffers for streaming mode DMA
689 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
690 * @sg: list of buffers
691 * @nents: number of buffers to map
692 * @dir: DMA transfer direction
694 * Map a set of buffers described by scatterlist in streaming mode for DMA.
695 * This is the scatter-gather version of the dma_map_single interface.
696 * Here the scatter gather list elements are each tagged with the
697 * appropriate dma address and length. They are obtained via
698 * sg_dma_{address,length}.
700 * Device ownership issues as mentioned for dma_map_single are the same
703 int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
704 enum dma_data_direction dir, struct dma_attrs *attrs)
706 struct dma_map_ops *ops = get_dma_ops(dev);
707 struct scatterlist *s;
710 for_each_sg(sg, s, nents, i) {
711 s->dma_address = ops->map_page(dev, sg_page(s), s->offset,
712 s->length, dir, attrs);
713 if (dma_mapping_error(dev, s->dma_address))
719 for_each_sg(sg, s, i, j)
720 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
725 * arm_dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
726 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
727 * @sg: list of buffers
728 * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
729 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
731 * Unmap a set of streaming mode DMA translations. Again, CPU access
732 * rules concerning calls here are the same as for dma_unmap_single().
734 void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
735 enum dma_data_direction dir, struct dma_attrs *attrs)
737 struct dma_map_ops *ops = get_dma_ops(dev);
738 struct scatterlist *s;
742 for_each_sg(sg, s, nents, i)
743 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
747 * arm_dma_sync_sg_for_cpu
748 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
749 * @sg: list of buffers
750 * @nents: number of buffers to map (returned from dma_map_sg)
751 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
753 void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
754 int nents, enum dma_data_direction dir)
756 struct dma_map_ops *ops = get_dma_ops(dev);
757 struct scatterlist *s;
760 for_each_sg(sg, s, nents, i)
761 ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length,
766 * arm_dma_sync_sg_for_device
767 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
768 * @sg: list of buffers
769 * @nents: number of buffers to map (returned from dma_map_sg)
770 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
772 void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
773 int nents, enum dma_data_direction dir)
775 struct dma_map_ops *ops = get_dma_ops(dev);
776 struct scatterlist *s;
779 for_each_sg(sg, s, nents, i)
780 ops->sync_single_for_device(dev, sg_dma_address(s), s->length,
785 * Return whether the given device DMA address mask can be supported
786 * properly. For example, if your device can only drive the low 24-bits
787 * during bus mastering, then you would pass 0x00ffffff as the mask
790 int dma_supported(struct device *dev, u64 mask)
792 if (mask < (u64)arm_dma_limit)
796 EXPORT_SYMBOL(dma_supported);
798 static int arm_dma_set_mask(struct device *dev, u64 dma_mask)
800 if (!dev->dma_mask || !dma_supported(dev, dma_mask))
803 *dev->dma_mask = dma_mask;
808 #define PREALLOC_DMA_DEBUG_ENTRIES 4096
810 static int __init dma_debug_do_init(void)
813 arm_vmregion_create_proc("dma-mappings", &consistent_head);
815 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
818 fs_initcall(dma_debug_do_init);