2 * omap iommu: simple virtual address space management
4 * Copyright (C) 2008-2009 Nokia Corporation
6 * Written by Hiroshi DOYU <Hiroshi.DOYU@nokia.com>
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
13 #include <linux/err.h>
14 #include <linux/slab.h>
15 #include <linux/vmalloc.h>
16 #include <linux/device.h>
17 #include <linux/scatterlist.h>
19 #include <asm/cacheflush.h>
20 #include <asm/mach/map.h>
22 #include <plat/iommu.h>
23 #include <plat/iovmm.h>
25 #include "iopgtable.h"
28 * A device driver needs to create address mappings between:
30 * - iommu/device address
32 * - mpu virtual address
34 * There are 4 possible patterns for them:
36 * |iova/ mapping iommu_ page
37 * | da pa va (d)-(p)-(v) function type
38 * ---------------------------------------------------------------------------
39 * 1 | c c c 1 - 1 - 1 _kmap() / _kunmap() s
40 * 2 | c c,a c 1 - 1 - 1 _kmalloc()/ _kfree() s
41 * 3 | c d c 1 - n - 1 _vmap() / _vunmap() s
42 * 4 | c d,a c 1 - n - 1 _vmalloc()/ _vfree() n*
45 * 'iova': device iommu virtual address
46 * 'da': alias of 'iova'
47 * 'pa': physical address
48 * 'va': mpu virtual address
50 * 'c': contiguous memory area
51 * 'd': discontiguous memory area
52 * 'a': anonymous memory allocation
53 * '()': optional feature
55 * 'n': a normal page(4KB) size is used.
56 * 's': multiple iommu superpage(16MB, 1MB, 64KB, 4KB) size is used.
58 * '*': not yet, but feasible.
61 static struct kmem_cache *iovm_area_cachep;
63 /* return total bytes of sg buffers */
64 static size_t sgtable_len(const struct sg_table *sgt)
66 unsigned int i, total = 0;
67 struct scatterlist *sg;
72 for_each_sg(sgt->sgl, sg, sgt->nents, i) {
75 bytes = sg_dma_len(sg);
77 if (!iopgsz_ok(bytes)) {
78 pr_err("%s: sg[%d] not iommu pagesize(%x)\n",
88 #define sgtable_ok(x) (!!sgtable_len(x))
90 static unsigned max_alignment(u32 addr)
93 unsigned pagesize[] = { SZ_16M, SZ_1M, SZ_64K, SZ_4K, };
94 for (i = 0; i < ARRAY_SIZE(pagesize) && addr & (pagesize[i] - 1); i++)
96 return (i < ARRAY_SIZE(pagesize)) ? pagesize[i] : 0;
100 * calculate the optimal number sg elements from total bytes based on
103 static unsigned sgtable_nents(size_t bytes, u32 da, u32 pa)
105 unsigned nr_entries = 0, ent_sz;
107 if (!IS_ALIGNED(bytes, PAGE_SIZE)) {
108 pr_err("%s: wrong size %08x\n", __func__, bytes);
113 ent_sz = max_alignment(da | pa);
114 ent_sz = min_t(unsigned, ent_sz, iopgsz_max(bytes));
124 /* allocate and initialize sg_table header(a kind of 'superblock') */
125 static struct sg_table *sgtable_alloc(const size_t bytes, u32 flags,
128 unsigned int nr_entries;
130 struct sg_table *sgt;
133 return ERR_PTR(-EINVAL);
135 if (!IS_ALIGNED(bytes, PAGE_SIZE))
136 return ERR_PTR(-EINVAL);
138 if (flags & IOVMF_LINEAR) {
139 nr_entries = sgtable_nents(bytes, da, pa);
141 return ERR_PTR(-EINVAL);
143 nr_entries = bytes / PAGE_SIZE;
145 sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
147 return ERR_PTR(-ENOMEM);
149 err = sg_alloc_table(sgt, nr_entries, GFP_KERNEL);
155 pr_debug("%s: sgt:%p(%d entries)\n", __func__, sgt, nr_entries);
160 /* free sg_table header(a kind of superblock) */
161 static void sgtable_free(struct sg_table *sgt)
169 pr_debug("%s: sgt:%p\n", __func__, sgt);
172 /* map 'sglist' to a contiguous mpu virtual area and return 'va' */
173 static void *vmap_sg(const struct sg_table *sgt)
178 struct scatterlist *sg;
179 struct vm_struct *new;
180 const struct mem_type *mtype;
182 mtype = get_mem_type(MT_DEVICE);
184 return ERR_PTR(-EINVAL);
186 total = sgtable_len(sgt);
188 return ERR_PTR(-EINVAL);
190 new = __get_vm_area(total, VM_IOREMAP, VMALLOC_START, VMALLOC_END);
192 return ERR_PTR(-ENOMEM);
195 for_each_sg(sgt->sgl, sg, sgt->nents, i) {
201 bytes = sg_dma_len(sg);
203 BUG_ON(bytes != PAGE_SIZE);
205 err = ioremap_page(va, pa, mtype);
212 flush_cache_vmap((unsigned long)new->addr,
213 (unsigned long)(new->addr + total));
217 WARN_ON(1); /* FIXME: cleanup some mpu mappings */
219 return ERR_PTR(-EAGAIN);
222 static inline void vunmap_sg(const void *va)
227 static struct iovm_struct *__find_iovm_area(struct iommu *obj, const u32 da)
229 struct iovm_struct *tmp;
231 list_for_each_entry(tmp, &obj->mmap, list) {
232 if ((da >= tmp->da_start) && (da < tmp->da_end)) {
235 len = tmp->da_end - tmp->da_start;
237 dev_dbg(obj->dev, "%s: %08x-%08x-%08x(%x) %08x\n",
238 __func__, tmp->da_start, da, tmp->da_end, len,
249 * find_iovm_area - find iovma which includes @da
250 * @da: iommu device virtual address
252 * Find the existing iovma starting at @da
254 struct iovm_struct *find_iovm_area(struct iommu *obj, u32 da)
256 struct iovm_struct *area;
258 mutex_lock(&obj->mmap_lock);
259 area = __find_iovm_area(obj, da);
260 mutex_unlock(&obj->mmap_lock);
264 EXPORT_SYMBOL_GPL(find_iovm_area);
267 * This finds the hole(area) which fits the requested address and len
268 * in iovmas mmap, and returns the new allocated iovma.
270 static struct iovm_struct *alloc_iovm_area(struct iommu *obj, u32 da,
271 size_t bytes, u32 flags)
273 struct iovm_struct *new, *tmp;
274 u32 start, prev_end, alignement;
277 return ERR_PTR(-EINVAL);
280 alignement = PAGE_SIZE;
282 if (flags & IOVMF_DA_ANON) {
283 start = obj->da_start;
285 if (flags & IOVMF_LINEAR)
286 alignement = iopgsz_max(bytes);
287 start = roundup(start, alignement);
288 } else if (start < obj->da_start || start > obj->da_end ||
289 obj->da_end - start < bytes) {
290 return ERR_PTR(-EINVAL);
294 if (list_empty(&obj->mmap))
298 list_for_each_entry(tmp, &obj->mmap, list) {
300 if (prev_end > start)
303 if (tmp->da_start > start && (tmp->da_start - start) >= bytes)
306 if (tmp->da_end >= start && flags & IOVMF_DA_ANON)
307 start = roundup(tmp->da_end + 1, alignement);
309 prev_end = tmp->da_end;
312 if ((start >= prev_end) && (obj->da_end - start >= bytes))
315 dev_dbg(obj->dev, "%s: no space to fit %08x(%x) flags: %08x\n",
316 __func__, da, bytes, flags);
318 return ERR_PTR(-EINVAL);
321 new = kmem_cache_zalloc(iovm_area_cachep, GFP_KERNEL);
323 return ERR_PTR(-ENOMEM);
326 new->da_start = start;
327 new->da_end = start + bytes;
331 * keep ascending order of iovmas
334 list_add_tail(&new->list, &tmp->list);
336 list_add(&new->list, &obj->mmap);
338 dev_dbg(obj->dev, "%s: found %08x-%08x-%08x(%x) %08x\n",
339 __func__, new->da_start, start, new->da_end, bytes, flags);
344 static void free_iovm_area(struct iommu *obj, struct iovm_struct *area)
348 BUG_ON(!obj || !area);
350 bytes = area->da_end - area->da_start;
352 dev_dbg(obj->dev, "%s: %08x-%08x(%x) %08x\n",
353 __func__, area->da_start, area->da_end, bytes, area->flags);
355 list_del(&area->list);
356 kmem_cache_free(iovm_area_cachep, area);
360 * da_to_va - convert (d) to (v)
361 * @obj: objective iommu
362 * @da: iommu device virtual address
363 * @va: mpu virtual address
365 * Returns mpu virtual addr which corresponds to a given device virtual addr
367 void *da_to_va(struct iommu *obj, u32 da)
370 struct iovm_struct *area;
372 mutex_lock(&obj->mmap_lock);
374 area = __find_iovm_area(obj, da);
376 dev_dbg(obj->dev, "%s: no da area(%08x)\n", __func__, da);
381 mutex_unlock(&obj->mmap_lock);
385 EXPORT_SYMBOL_GPL(da_to_va);
387 static void sgtable_fill_vmalloc(struct sg_table *sgt, void *_va)
390 struct scatterlist *sg;
394 for_each_sg(sgt->sgl, sg, sgt->nents, i) {
396 const size_t bytes = PAGE_SIZE;
399 * iommu 'superpage' isn't supported with 'iommu_vmalloc()'
401 pg = vmalloc_to_page(va);
403 sg_set_page(sg, pg, bytes, 0);
408 va_end = _va + PAGE_SIZE * i;
411 static inline void sgtable_drain_vmalloc(struct sg_table *sgt)
414 * Actually this is not necessary at all, just exists for
415 * consistency of the code readability.
420 static void sgtable_fill_kmalloc(struct sg_table *sgt, u32 pa, u32 da,
424 struct scatterlist *sg;
427 va = phys_to_virt(pa);
429 for_each_sg(sgt->sgl, sg, sgt->nents, i) {
432 bytes = max_alignment(da | pa);
433 bytes = min_t(unsigned, bytes, iopgsz_max(len));
435 BUG_ON(!iopgsz_ok(bytes));
437 sg_set_buf(sg, phys_to_virt(pa), bytes);
439 * 'pa' is cotinuous(linear).
448 static inline void sgtable_drain_kmalloc(struct sg_table *sgt)
451 * Actually this is not necessary at all, just exists for
452 * consistency of the code readability
457 /* create 'da' <-> 'pa' mapping from 'sgt' */
458 static int map_iovm_area(struct iommu *obj, struct iovm_struct *new,
459 const struct sg_table *sgt, u32 flags)
463 struct scatterlist *sg;
464 u32 da = new->da_start;
469 BUG_ON(!sgtable_ok(sgt));
471 for_each_sg(sgt->sgl, sg, sgt->nents, i) {
475 struct iotlb_entry e;
478 bytes = sg_dma_len(sg);
480 flags &= ~IOVMF_PGSZ_MASK;
481 pgsz = bytes_to_iopgsz(bytes);
486 pr_debug("%s: [%d] %08x %08x(%x)\n", __func__,
489 iotlb_init_entry(&e, da, pa, flags);
490 err = iopgtable_store_entry(obj, &e);
501 for_each_sg(sgt->sgl, sg, i, j) {
504 bytes = iopgtable_clear_entry(obj, da);
506 BUG_ON(!iopgsz_ok(bytes));
513 /* release 'da' <-> 'pa' mapping */
514 static void unmap_iovm_area(struct iommu *obj, struct iovm_struct *area)
517 size_t total = area->da_end - area->da_start;
519 BUG_ON((!total) || !IS_ALIGNED(total, PAGE_SIZE));
521 start = area->da_start;
525 bytes = iopgtable_clear_entry(obj, start);
529 dev_dbg(obj->dev, "%s: unmap %08x(%x) %08x\n",
530 __func__, start, bytes, area->flags);
532 BUG_ON(!IS_ALIGNED(bytes, PAGE_SIZE));
540 /* template function for all unmapping */
541 static struct sg_table *unmap_vm_area(struct iommu *obj, const u32 da,
542 void (*fn)(const void *), u32 flags)
544 struct sg_table *sgt = NULL;
545 struct iovm_struct *area;
547 if (!IS_ALIGNED(da, PAGE_SIZE)) {
548 dev_err(obj->dev, "%s: alignment err(%08x)\n", __func__, da);
552 mutex_lock(&obj->mmap_lock);
554 area = __find_iovm_area(obj, da);
556 dev_dbg(obj->dev, "%s: no da area(%08x)\n", __func__, da);
560 if ((area->flags & flags) != flags) {
561 dev_err(obj->dev, "%s: wrong flags(%08x)\n", __func__,
565 sgt = (struct sg_table *)area->sgt;
567 unmap_iovm_area(obj, area);
571 dev_dbg(obj->dev, "%s: %08x-%08x-%08x(%x) %08x\n", __func__,
572 area->da_start, da, area->da_end,
573 area->da_end - area->da_start, area->flags);
575 free_iovm_area(obj, area);
577 mutex_unlock(&obj->mmap_lock);
582 static u32 map_iommu_region(struct iommu *obj, u32 da,
583 const struct sg_table *sgt, void *va, size_t bytes, u32 flags)
586 struct iovm_struct *new;
588 mutex_lock(&obj->mmap_lock);
590 new = alloc_iovm_area(obj, da, bytes, flags);
593 goto err_alloc_iovma;
598 if (map_iovm_area(obj, new, sgt, new->flags))
601 mutex_unlock(&obj->mmap_lock);
603 dev_dbg(obj->dev, "%s: da:%08x(%x) flags:%08x va:%p\n",
604 __func__, new->da_start, bytes, new->flags, va);
606 return new->da_start;
609 free_iovm_area(obj, new);
611 mutex_unlock(&obj->mmap_lock);
615 static inline u32 __iommu_vmap(struct iommu *obj, u32 da,
616 const struct sg_table *sgt, void *va, size_t bytes, u32 flags)
618 return map_iommu_region(obj, da, sgt, va, bytes, flags);
622 * iommu_vmap - (d)-(p)-(v) address mapper
623 * @obj: objective iommu
624 * @sgt: address of scatter gather table
625 * @flags: iovma and page property
627 * Creates 1-n-1 mapping with given @sgt and returns @da.
628 * All @sgt element must be io page size aligned.
630 u32 iommu_vmap(struct iommu *obj, u32 da, const struct sg_table *sgt,
636 if (!obj || !obj->dev || !sgt)
639 bytes = sgtable_len(sgt);
642 bytes = PAGE_ALIGN(bytes);
644 if (flags & IOVMF_MMIO) {
650 flags &= IOVMF_HW_MASK;
651 flags |= IOVMF_DISCONT;
653 flags |= (da ? IOVMF_DA_FIXED : IOVMF_DA_ANON);
655 da = __iommu_vmap(obj, da, sgt, va, bytes, flags);
656 if (IS_ERR_VALUE(da))
661 EXPORT_SYMBOL_GPL(iommu_vmap);
664 * iommu_vunmap - release virtual mapping obtained by 'iommu_vmap()'
665 * @obj: objective iommu
666 * @da: iommu device virtual address
668 * Free the iommu virtually contiguous memory area starting at
669 * @da, which was returned by 'iommu_vmap()'.
671 struct sg_table *iommu_vunmap(struct iommu *obj, u32 da)
673 struct sg_table *sgt;
675 * 'sgt' is allocated before 'iommu_vmalloc()' is called.
676 * Just returns 'sgt' to the caller to free
678 sgt = unmap_vm_area(obj, da, vunmap_sg, IOVMF_DISCONT | IOVMF_MMIO);
680 dev_dbg(obj->dev, "%s: No sgt\n", __func__);
683 EXPORT_SYMBOL_GPL(iommu_vunmap);
686 * iommu_vmalloc - (d)-(p)-(v) address allocator and mapper
687 * @obj: objective iommu
688 * @da: contiguous iommu virtual memory
689 * @bytes: allocation size
690 * @flags: iovma and page property
692 * Allocate @bytes linearly and creates 1-n-1 mapping and returns
693 * @da again, which might be adjusted if 'IOVMF_DA_ANON' is set.
695 u32 iommu_vmalloc(struct iommu *obj, u32 da, size_t bytes, u32 flags)
698 struct sg_table *sgt;
700 if (!obj || !obj->dev || !bytes)
703 bytes = PAGE_ALIGN(bytes);
709 flags &= IOVMF_HW_MASK;
710 flags |= IOVMF_DISCONT;
711 flags |= IOVMF_ALLOC;
712 flags |= (da ? IOVMF_DA_FIXED : IOVMF_DA_ANON);
714 sgt = sgtable_alloc(bytes, flags, da, 0);
719 sgtable_fill_vmalloc(sgt, va);
721 da = __iommu_vmap(obj, da, sgt, va, bytes, flags);
722 if (IS_ERR_VALUE(da))
728 sgtable_drain_vmalloc(sgt);
734 EXPORT_SYMBOL_GPL(iommu_vmalloc);
737 * iommu_vfree - release memory allocated by 'iommu_vmalloc()'
738 * @obj: objective iommu
739 * @da: iommu device virtual address
741 * Frees the iommu virtually continuous memory area starting at
742 * @da, as obtained from 'iommu_vmalloc()'.
744 void iommu_vfree(struct iommu *obj, const u32 da)
746 struct sg_table *sgt;
748 sgt = unmap_vm_area(obj, da, vfree, IOVMF_DISCONT | IOVMF_ALLOC);
750 dev_dbg(obj->dev, "%s: No sgt\n", __func__);
753 EXPORT_SYMBOL_GPL(iommu_vfree);
755 static u32 __iommu_kmap(struct iommu *obj, u32 da, u32 pa, void *va,
756 size_t bytes, u32 flags)
758 struct sg_table *sgt;
760 sgt = sgtable_alloc(bytes, flags, da, pa);
764 sgtable_fill_kmalloc(sgt, pa, da, bytes);
766 da = map_iommu_region(obj, da, sgt, va, bytes, flags);
767 if (IS_ERR_VALUE(da)) {
768 sgtable_drain_kmalloc(sgt);
776 * iommu_kmap - (d)-(p)-(v) address mapper
777 * @obj: objective iommu
778 * @da: contiguous iommu virtual memory
779 * @pa: contiguous physical memory
780 * @flags: iovma and page property
782 * Creates 1-1-1 mapping and returns @da again, which can be
783 * adjusted if 'IOVMF_DA_ANON' is set.
785 u32 iommu_kmap(struct iommu *obj, u32 da, u32 pa, size_t bytes,
790 if (!obj || !obj->dev || !bytes)
793 bytes = PAGE_ALIGN(bytes);
795 va = ioremap(pa, bytes);
799 flags &= IOVMF_HW_MASK;
800 flags |= IOVMF_LINEAR;
802 flags |= (da ? IOVMF_DA_FIXED : IOVMF_DA_ANON);
804 da = __iommu_kmap(obj, da, pa, va, bytes, flags);
805 if (IS_ERR_VALUE(da))
810 EXPORT_SYMBOL_GPL(iommu_kmap);
813 * iommu_kunmap - release virtual mapping obtained by 'iommu_kmap()'
814 * @obj: objective iommu
815 * @da: iommu device virtual address
817 * Frees the iommu virtually contiguous memory area starting at
818 * @da, which was passed to and was returned by'iommu_kmap()'.
820 void iommu_kunmap(struct iommu *obj, u32 da)
822 struct sg_table *sgt;
823 typedef void (*func_t)(const void *);
825 sgt = unmap_vm_area(obj, da, (func_t)iounmap,
826 IOVMF_LINEAR | IOVMF_MMIO);
828 dev_dbg(obj->dev, "%s: No sgt\n", __func__);
831 EXPORT_SYMBOL_GPL(iommu_kunmap);
834 * iommu_kmalloc - (d)-(p)-(v) address allocator and mapper
835 * @obj: objective iommu
836 * @da: contiguous iommu virtual memory
837 * @bytes: bytes for allocation
838 * @flags: iovma and page property
840 * Allocate @bytes linearly and creates 1-1-1 mapping and returns
841 * @da again, which might be adjusted if 'IOVMF_DA_ANON' is set.
843 u32 iommu_kmalloc(struct iommu *obj, u32 da, size_t bytes, u32 flags)
848 if (!obj || !obj->dev || !bytes)
851 bytes = PAGE_ALIGN(bytes);
853 va = kmalloc(bytes, GFP_KERNEL | GFP_DMA);
856 pa = virt_to_phys(va);
858 flags &= IOVMF_HW_MASK;
859 flags |= IOVMF_LINEAR;
860 flags |= IOVMF_ALLOC;
861 flags |= (da ? IOVMF_DA_FIXED : IOVMF_DA_ANON);
863 da = __iommu_kmap(obj, da, pa, va, bytes, flags);
864 if (IS_ERR_VALUE(da))
869 EXPORT_SYMBOL_GPL(iommu_kmalloc);
872 * iommu_kfree - release virtual mapping obtained by 'iommu_kmalloc()'
873 * @obj: objective iommu
874 * @da: iommu device virtual address
876 * Frees the iommu virtually contiguous memory area starting at
877 * @da, which was passed to and was returned by'iommu_kmalloc()'.
879 void iommu_kfree(struct iommu *obj, u32 da)
881 struct sg_table *sgt;
883 sgt = unmap_vm_area(obj, da, kfree, IOVMF_LINEAR | IOVMF_ALLOC);
885 dev_dbg(obj->dev, "%s: No sgt\n", __func__);
888 EXPORT_SYMBOL_GPL(iommu_kfree);
891 static int __init iovmm_init(void)
893 const unsigned long flags = SLAB_HWCACHE_ALIGN;
894 struct kmem_cache *p;
896 p = kmem_cache_create("iovm_area_cache", sizeof(struct iovm_struct), 0,
900 iovm_area_cachep = p;
904 module_init(iovmm_init);
906 static void __exit iovmm_exit(void)
908 kmem_cache_destroy(iovm_area_cachep);
910 module_exit(iovmm_exit);
912 MODULE_DESCRIPTION("omap iommu: simple virtual address space management");
913 MODULE_AUTHOR("Hiroshi DOYU <Hiroshi.DOYU@nokia.com>");
914 MODULE_LICENSE("GPL v2");