static void *__alloc_from_contiguous(struct device *dev, size_t size,
pgprot_t prot, struct page **ret_page,
- const void *caller);
+ const void *caller, bool want_vaddr);
static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
pgprot_t prot, struct page **ret_page,
- const void *caller);
+ const void *caller, bool want_vaddr);
static void *
__dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot,
if (dev_get_cma_area(NULL))
ptr = __alloc_from_contiguous(NULL, atomic_pool_size, prot,
- &page, atomic_pool_init);
+ &page, atomic_pool_init, true);
else
ptr = __alloc_remap_buffer(NULL, atomic_pool_size, gfp, prot,
- &page, atomic_pool_init);
+ &page, atomic_pool_init, true);
if (ptr) {
int ret;
static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
pgprot_t prot, struct page **ret_page,
- const void *caller)
+ const void *caller, bool want_vaddr)
{
struct page *page;
- void *ptr;
+ void *ptr = NULL;
page = __dma_alloc_buffer(dev, size, gfp);
if (!page)
return NULL;
+ if (!want_vaddr)
+ goto out;
ptr = __dma_alloc_remap(page, size, gfp, prot, caller);
if (!ptr) {
return NULL;
}
+ out:
*ret_page = page;
return ptr;
}
static void *__alloc_from_contiguous(struct device *dev, size_t size,
pgprot_t prot, struct page **ret_page,
- const void *caller)
+ const void *caller, bool want_vaddr)
{
unsigned long order = get_order(size);
size_t count = size >> PAGE_SHIFT;
struct page *page;
- void *ptr;
+ void *ptr = NULL;
page = dma_alloc_from_contiguous(dev, count, order);
if (!page)
__dma_clear_buffer(page, size);
+ if (!want_vaddr)
+ goto out;
+
if (PageHighMem(page)) {
ptr = __dma_alloc_remap(page, size, GFP_KERNEL, prot, caller);
if (!ptr) {
__dma_remap(page, size, prot);
ptr = page_address(page);
}
+
+ out:
*ret_page = page;
return ptr;
}
static void __free_from_contiguous(struct device *dev, struct page *page,
- void *cpu_addr, size_t size)
+ void *cpu_addr, size_t size, bool want_vaddr)
{
- if (PageHighMem(page))
- __dma_free_remap(cpu_addr, size);
- else
- __dma_remap(page, size, PAGE_KERNEL);
+ if (want_vaddr) {
+ if (PageHighMem(page))
+ __dma_free_remap(cpu_addr, size);
+ else
+ __dma_remap(page, size, PAGE_KERNEL);
+ }
dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);
}
#define nommu() 1
-#define __get_dma_pgprot(attrs, prot) __pgprot(0)
-#define __alloc_remap_buffer(dev, size, gfp, prot, ret, c) NULL
+#define __get_dma_pgprot(attrs, prot) __pgprot(0)
+#define __alloc_remap_buffer(dev, size, gfp, prot, ret, c, wv) NULL
#define __alloc_from_pool(size, ret_page) NULL
-#define __alloc_from_contiguous(dev, size, prot, ret, c) NULL
+#define __alloc_from_contiguous(dev, size, prot, ret, c, wv) NULL
#define __free_from_pool(cpu_addr, size) 0
-#define __free_from_contiguous(dev, page, cpu_addr, size) do { } while (0)
+#define __free_from_contiguous(dev, page, cpu_addr, size, wv) do { } while (0)
#define __dma_free_remap(cpu_addr, size) do { } while (0)
#endif /* CONFIG_MMU */
static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
- gfp_t gfp, pgprot_t prot, bool is_coherent, const void *caller)
+ gfp_t gfp, pgprot_t prot, bool is_coherent,
+ struct dma_attrs *attrs, const void *caller)
{
u64 mask = get_coherent_dma_mask(dev);
struct page *page = NULL;
void *addr;
+ bool want_vaddr;
#ifdef CONFIG_DMA_API_DEBUG
u64 limit = (mask + 1) & ~mask;
*handle = DMA_ERROR_CODE;
size = PAGE_ALIGN(size);
+ want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs);
if (is_coherent || nommu())
addr = __alloc_simple_buffer(dev, size, gfp, &page);
else if (!(gfp & __GFP_WAIT))
addr = __alloc_from_pool(size, &page);
else if (!dev_get_cma_area(dev))
- addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, caller);
+ addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, caller, want_vaddr);
else
- addr = __alloc_from_contiguous(dev, size, prot, &page, caller);
+ addr = __alloc_from_contiguous(dev, size, prot, &page, caller, want_vaddr);
- if (addr)
+ if (page)
*handle = pfn_to_dma(dev, page_to_pfn(page));
- return addr;
+ return want_vaddr ? addr : page;
}
/*
return memory;
return __dma_alloc(dev, size, handle, gfp, prot, false,
- __builtin_return_address(0));
+ attrs, __builtin_return_address(0));
}
static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
return memory;
return __dma_alloc(dev, size, handle, gfp, prot, true,
- __builtin_return_address(0));
+ attrs, __builtin_return_address(0));
}
/*
bool is_coherent)
{
struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
+ bool want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs);
if (dma_release_from_coherent(dev, get_order(size), cpu_addr))
return;
} else if (__free_from_pool(cpu_addr, size)) {
return;
} else if (!dev_get_cma_area(dev)) {
- __dma_free_remap(cpu_addr, size);
+ if (want_vaddr)
+ __dma_free_remap(cpu_addr, size);
__dma_free_buffer(page, size);
} else {
/*
* Non-atomic allocations cannot be freed with IRQs disabled
*/
WARN_ON(irqs_disabled());
- __free_from_contiguous(dev, page, cpu_addr, size);
+ __free_from_contiguous(dev, page, cpu_addr, size, want_vaddr);
}
}
gfp |= __GFP_NOWARN | __GFP_HIGHMEM;
while (count) {
- int j, order = __fls(count);
+ int j, order;
+
+ for (order = __fls(count); order > 0; --order) {
+ /*
+ * We do not want OOM killer to be invoked as long
+ * as we can fall back to single pages, so we force
+ * __GFP_NORETRY for orders higher than zero.
+ */
+ pages[i] = alloc_pages(gfp | __GFP_NORETRY, order);
+ if (pages[i])
+ break;
+ }
- pages[i] = alloc_pages(gfp, order);
- while (!pages[i] && order)
- pages[i] = alloc_pages(gfp, --order);
- if (!pages[i])
- goto error;
+ if (!pages[i]) {
+ /*
+ * Fall back to single page allocation.
+ * Might invoke OOM killer as last resort.
+ */
+ pages[i] = alloc_pages(gfp, 0);
+ if (!pages[i])
+ goto error;
+ }
if (order) {
split_page(pages[i], order);
static dma_addr_t
__iommu_create_mapping(struct device *dev, struct page **pages, size_t size)
{
- struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+ struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
dma_addr_t dma_addr, iova;
int i, ret = DMA_ERROR_CODE;
static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size)
{
- struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+ struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
/*
* add optional in-page offset from iova to size and align
enum dma_data_direction dir, struct dma_attrs *attrs,
bool is_coherent)
{
- struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+ struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
dma_addr_t iova, iova_base;
int ret = 0;
unsigned int count;
unsigned long offset, size_t size, enum dma_data_direction dir,
struct dma_attrs *attrs)
{
- struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+ struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
dma_addr_t dma_addr;
int ret, prot, len = PAGE_ALIGN(size + offset);
size_t size, enum dma_data_direction dir,
struct dma_attrs *attrs)
{
- struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+ struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
dma_addr_t iova = handle & PAGE_MASK;
int offset = handle & ~PAGE_MASK;
int len = PAGE_ALIGN(size + offset);
size_t size, enum dma_data_direction dir,
struct dma_attrs *attrs)
{
- struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+ struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
dma_addr_t iova = handle & PAGE_MASK;
struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
int offset = handle & ~PAGE_MASK;
static void arm_iommu_sync_single_for_cpu(struct device *dev,
dma_addr_t handle, size_t size, enum dma_data_direction dir)
{
- struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+ struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
dma_addr_t iova = handle & PAGE_MASK;
struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
unsigned int offset = handle & ~PAGE_MASK;
static void arm_iommu_sync_single_for_device(struct device *dev,
dma_addr_t handle, size_t size, enum dma_data_direction dir)
{
- struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+ struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
dma_addr_t iova = handle & PAGE_MASK;
struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
unsigned int offset = handle & ~PAGE_MASK;
return err;
kref_get(&mapping->kref);
- dev->archdata.mapping = mapping;
+ to_dma_iommu_mapping(dev) = mapping;
pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev));
return 0;
iommu_detach_device(mapping->domain, dev);
kref_put(&mapping->kref, release_iommu_mapping);
- dev->archdata.mapping = NULL;
+ to_dma_iommu_mapping(dev) = NULL;
pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev));
}
static void arm_teardown_iommu_dma_ops(struct device *dev)
{
- struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+ struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
if (!mapping)
return;