#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
-#ifndef PHYSICAL_PAGE_MASK
-#define PHYSICAL_PAGE_MASK PAGE_MASK
-#endif
/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
are never going to work. */
return NULL;
}
-/* clear one page's page table */
-static void dma_pte_clear_one(struct dmar_domain *domain, unsigned long pfn)
-{
- struct dma_pte *pte = NULL;
-
- /* get last level pte */
- pte = dma_pfn_level_pte(domain, pfn, 1);
-
- if (pte) {
- dma_clear_pte(pte);
- domain_flush_cache(domain, pte, sizeof(*pte));
- }
-}
-
/* clear last level pte, a tlb flush should be followed */
static void dma_pte_clear_range(struct dmar_domain *domain,
unsigned long start_pfn,
unsigned long last_pfn)
{
int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
+ struct dma_pte *first_pte, *pte;
BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
/* we don't need lock here; nobody else touches the iova range */
while (start_pfn <= last_pfn) {
- dma_pte_clear_one(domain, start_pfn);
- start_pfn++;
+ first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1);
+ if (!pte) {
+ start_pfn = align_to_level(start_pfn + 1, 2);
+ continue;
+ }
+ while (start_pfn <= last_pfn &&
+ (unsigned long)pte >> VTD_PAGE_SHIFT ==
+ (unsigned long)first_pte >> VTD_PAGE_SHIFT) {
+ dma_clear_pte(pte);
+ start_pfn++;
+ pte++;
+ }
+ domain_flush_cache(domain, first_pte,
+ (void *)pte - (void *)first_pte);
}
}
if (tmp + level_size(level) - 1 > last_pfn)
return;
- while (tmp <= last_pfn) {
+ while (tmp + level_size(level) - 1 <= last_pfn) {
pte = dma_pfn_level_pte(domain, tmp, level);
if (pte) {
free_pgtable_page(
}
static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
- u64 addr, unsigned int pages)
+ unsigned long pfn, unsigned int pages)
{
unsigned int mask = ilog2(__roundup_pow_of_two(pages));
+ uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
- BUG_ON(addr & (~VTD_PAGE_MASK));
BUG_ON(pages == 0);
/*
struct pci_dev *pdev = NULL;
struct iova *iova;
int i;
- u64 addr, size;
init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
r = &pdev->resource[i];
if (!r->flags || !(r->flags & IORESOURCE_MEM))
continue;
- addr = r->start;
- addr &= PHYSICAL_PAGE_MASK;
- size = r->end - addr;
- size = PAGE_ALIGN(size);
- iova = reserve_iova(&reserved_iova_list, IOVA_PFN(addr),
- IOVA_PFN(size + addr) - 1);
+ iova = reserve_iova(&reserved_iova_list,
+ IOVA_PFN(r->start),
+ IOVA_PFN(r->end));
if (!iova)
printk(KERN_ERR "Reserve iova failed\n");
}
tmp->devfn);
}
-static int
-domain_page_mapping(struct dmar_domain *domain, dma_addr_t iova,
- u64 hpa, size_t size, int prot)
+static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
+ struct scatterlist *sg, unsigned long phys_pfn,
+ unsigned long nr_pages, int prot)
{
- u64 start_pfn, end_pfn;
- struct dma_pte *pte;
- int index;
- int addr_width = agaw_to_width(domain->agaw);
+ struct dma_pte *first_pte = NULL, *pte = NULL;
+ phys_addr_t uninitialized_var(pteval);
+ int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
+ unsigned long sg_res;
- BUG_ON(hpa >> addr_width);
+ BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
return -EINVAL;
- iova &= PAGE_MASK;
- start_pfn = ((u64)hpa) >> VTD_PAGE_SHIFT;
- end_pfn = (VTD_PAGE_ALIGN(((u64)hpa) + size)) >> VTD_PAGE_SHIFT;
- index = 0;
- while (start_pfn < end_pfn) {
- pte = pfn_to_dma_pte(domain, (iova >> VTD_PAGE_SHIFT) + index);
- if (!pte)
- return -ENOMEM;
+
+ prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
+
+ if (sg)
+ sg_res = 0;
+ else {
+ sg_res = nr_pages + 1;
+ pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
+ }
+
+ while (nr_pages--) {
+ if (!sg_res) {
+ sg_res = (sg->offset + sg->length + VTD_PAGE_SIZE - 1) >> VTD_PAGE_SHIFT;
+ sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
+ sg->dma_length = sg->length;
+ pteval = page_to_phys(sg_page(sg)) | prot;
+ }
+ if (!pte) {
+ first_pte = pte = pfn_to_dma_pte(domain, iov_pfn);
+ if (!pte)
+ return -ENOMEM;
+ }
/* We don't need lock here, nobody else
* touches the iova range
*/
- BUG_ON(dma_pte_addr(pte));
- dma_set_pte_pfn(pte, start_pfn);
- dma_set_pte_prot(pte, prot);
- if (prot & DMA_PTE_SNP)
- dma_set_pte_snp(pte);
- domain_flush_cache(domain, pte, sizeof(*pte));
- start_pfn++;
- index++;
+ if (unlikely(dma_pte_addr(pte))) {
+ static int dumps = 5;
+ printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx)\n",
+ iov_pfn, pte->val);
+ if (dumps) {
+ dumps--;
+ debug_dma_dump_mappings(NULL);
+ }
+ WARN_ON(1);
+ }
+ pte->val = pteval;
+ pte++;
+ if (!nr_pages ||
+ (unsigned long)pte >> VTD_PAGE_SHIFT !=
+ (unsigned long)first_pte >> VTD_PAGE_SHIFT) {
+ domain_flush_cache(domain, first_pte,
+ (void *)pte - (void *)first_pte);
+ pte = NULL;
+ }
+ iov_pfn++;
+ pteval += VTD_PAGE_SIZE;
+ sg_res--;
+ if (!sg_res)
+ sg = sg_next(sg);
}
return 0;
}
+static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
+ struct scatterlist *sg, unsigned long nr_pages,
+ int prot)
+{
+ return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
+}
+
+static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
+ unsigned long phys_pfn, unsigned long nr_pages,
+ int prot)
+{
+ return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
+}
+
static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
{
if (!iommu)
unsigned long long start,
unsigned long long end)
{
- unsigned long size;
- unsigned long long base;
+ unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
+ unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
- /* The address might not be aligned */
- base = start & PAGE_MASK;
- size = end - base;
- size = PAGE_ALIGN(size);
- if (!reserve_iova(&domain->iovad, IOVA_PFN(base),
- IOVA_PFN(base + size) - 1)) {
+ if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
+ dma_to_mm_pfn(last_vpfn))) {
printk(KERN_ERR "IOMMU: reserve iova failed\n");
return -ENOMEM;
}
- pr_debug("Mapping reserved region %lx@%llx for domain %d\n",
- size, base, domain->id);
+ pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
+ start, end, domain->id);
/*
* RMRR range might have overlap with physical memory range,
* clear it first
*/
- dma_pte_clear_range(domain, base >> VTD_PAGE_SHIFT,
- (base + size - 1) >> VTD_PAGE_SHIFT);
+ dma_pte_clear_range(domain, first_vpfn, last_vpfn);
- return domain_page_mapping(domain, base, base, size,
- DMA_PTE_READ|DMA_PTE_WRITE);
+ return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
+ last_vpfn - first_vpfn + 1,
+ DMA_PTE_READ|DMA_PTE_WRITE);
}
static int iommu_prepare_identity_map(struct pci_dev *pdev,
return ret;
}
-static inline u64 aligned_size(u64 host_addr, size_t size)
+static inline unsigned long aligned_nrpages(unsigned long host_addr,
+ size_t size)
{
- u64 addr;
- addr = (host_addr & (~PAGE_MASK)) + size;
- return PAGE_ALIGN(addr);
-}
-
-struct iova *
-iommu_alloc_iova(struct dmar_domain *domain, size_t size, u64 end)
-{
- struct iova *piova;
-
- /* Make sure it's in range */
- end = min_t(u64, DOMAIN_MAX_ADDR(domain->gaw), end);
- if (!size || (IOVA_START_ADDR + size > end))
- return NULL;
+ host_addr &= ~PAGE_MASK;
+ host_addr += size + PAGE_SIZE - 1;
- piova = alloc_iova(&domain->iovad,
- size >> PAGE_SHIFT, IOVA_PFN(end), 1);
- return piova;
+ return host_addr >> VTD_PAGE_SHIFT;
}
-static struct iova *
-__intel_alloc_iova(struct device *dev, struct dmar_domain *domain,
- size_t size, u64 dma_mask)
+static struct iova *intel_alloc_iova(struct device *dev,
+ struct dmar_domain *domain,
+ unsigned long nrpages, uint64_t dma_mask)
{
struct pci_dev *pdev = to_pci_dev(dev);
struct iova *iova = NULL;
- if (dma_mask <= DMA_BIT_MASK(32) || dmar_forcedac)
- iova = iommu_alloc_iova(domain, size, dma_mask);
- else {
+ /* Restrict dma_mask to the width that the iommu can handle */
+ dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
+
+ if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
/*
* First try to allocate an io virtual address in
* DMA_BIT_MASK(32) and if that fails then try allocating
* from higher range
*/
- iova = iommu_alloc_iova(domain, size, DMA_BIT_MASK(32));
- if (!iova)
- iova = iommu_alloc_iova(domain, size, dma_mask);
- }
-
- if (!iova) {
- printk(KERN_ERR"Allocating iova for %s failed", pci_name(pdev));
+ iova = alloc_iova(&domain->iovad, nrpages,
+ IOVA_PFN(DMA_BIT_MASK(32)), 1);
+ if (iova)
+ return iova;
+ }
+ iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
+ if (unlikely(!iova)) {
+ printk(KERN_ERR "Allocating %ld-page iova for %s failed",
+ nrpages, pci_name(pdev));
return NULL;
}
return 0;
iommu = domain_get_iommu(domain);
- size = aligned_size((u64)paddr, size);
+ size = aligned_nrpages(paddr, size);
- iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
+ iova = intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
if (!iova)
goto error;
- start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
-
/*
* Check if DMAR supports zero-length reads on write only
* mappings..
* might have two guest_addr mapping to the same host paddr, but this
* is not a big problem
*/
- ret = domain_page_mapping(domain, start_paddr,
- ((u64)paddr) & PHYSICAL_PAGE_MASK,
- size, prot);
+ ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
+ paddr >> VTD_PAGE_SHIFT, size, prot);
if (ret)
goto error;
/* it's a non-present to present mapping. Only flush if caching mode */
if (cap_caching_mode(iommu->cap))
- iommu_flush_iotlb_psi(iommu, 0, start_paddr,
- size >> VTD_PAGE_SHIFT);
+ iommu_flush_iotlb_psi(iommu, 0, mm_to_dma_pfn(iova->pfn_lo), size);
else
iommu_flush_write_buffer(iommu);
- return start_paddr + ((u64)paddr & (~PAGE_MASK));
+ start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
+ start_paddr += paddr & ~PAGE_MASK;
+ return start_paddr;
error:
if (iova)
dma_pte_free_pagetable(domain, start_pfn, last_pfn);
if (intel_iommu_strict) {
- iommu_flush_iotlb_psi(iommu, domain->id,
- start_pfn << VTD_PAGE_SHIFT,
+ iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
last_pfn - start_pfn + 1);
/* free iova */
__free_iova(&domain->iovad, iova);
/* free page tables */
dma_pte_free_pagetable(domain, start_pfn, last_pfn);
- iommu_flush_iotlb_psi(iommu, domain->id,
- start_pfn << VTD_PAGE_SHIFT,
+ iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
(last_pfn - start_pfn + 1));
/* free iova */
static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
enum dma_data_direction dir, struct dma_attrs *attrs)
{
- phys_addr_t addr;
int i;
struct pci_dev *pdev = to_pci_dev(hwdev);
struct dmar_domain *domain;
size_t size = 0;
int prot = 0;
- size_t offset = 0;
+ size_t offset_pfn = 0;
struct iova *iova = NULL;
int ret;
struct scatterlist *sg;
- unsigned long start_addr;
+ unsigned long start_vpfn;
struct intel_iommu *iommu;
BUG_ON(dir == DMA_NONE);
iommu = domain_get_iommu(domain);
- for_each_sg(sglist, sg, nelems, i) {
- addr = page_to_phys(sg_page(sg)) + sg->offset;
- size += aligned_size((u64)addr, sg->length);
- }
+ for_each_sg(sglist, sg, nelems, i)
+ size += aligned_nrpages(sg->offset, sg->length);
- iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
+ iova = intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
if (!iova) {
sglist->dma_length = 0;
return 0;
if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
prot |= DMA_PTE_WRITE;
- start_addr = iova->pfn_lo << PAGE_SHIFT;
- offset = 0;
- for_each_sg(sglist, sg, nelems, i) {
- addr = page_to_phys(sg_page(sg)) + sg->offset;
- size = aligned_size((u64)addr, sg->length);
- ret = domain_page_mapping(domain, start_addr + offset,
- ((u64)addr) & PHYSICAL_PAGE_MASK,
- size, prot);
- if (ret) {
- /* clear the page */
- dma_pte_clear_range(domain,
- start_addr >> VTD_PAGE_SHIFT,
- (start_addr + offset - 1) >> VTD_PAGE_SHIFT);
- /* free page tables */
- dma_pte_free_pagetable(domain, start_addr >> VTD_PAGE_SHIFT,
- (start_addr + offset - 1) >> VTD_PAGE_SHIFT);
- /* free iova */
- __free_iova(&domain->iovad, iova);
- return 0;
- }
- sg->dma_address = start_addr + offset +
- ((u64)addr & (~PAGE_MASK));
- sg->dma_length = sg->length;
- offset += size;
+ start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
+
+ ret = domain_sg_mapping(domain, start_vpfn, sglist, mm_to_dma_pfn(size), prot);
+ if (unlikely(ret)) {
+ /* clear the page */
+ dma_pte_clear_range(domain, start_vpfn,
+ start_vpfn + size - 1);
+ /* free page tables */
+ dma_pte_free_pagetable(domain, start_vpfn,
+ start_vpfn + size - 1);
+ /* free iova */
+ __free_iova(&domain->iovad, iova);
+ return 0;
}
/* it's a non-present to present mapping. Only flush if caching mode */
if (cap_caching_mode(iommu->cap))
- iommu_flush_iotlb_psi(iommu, 0, start_addr,
- offset >> VTD_PAGE_SHIFT);
+ iommu_flush_iotlb_psi(iommu, 0, start_vpfn, offset_pfn);
else
iommu_flush_write_buffer(iommu);
}
dmar_domain->max_addr = max_addr;
}
-
- ret = domain_page_mapping(dmar_domain, iova, hpa, size, prot);
+ /* Round up size to next multiple of PAGE_SIZE, if it and
+ the low bits of hpa would take us onto the next page */
+ size = aligned_nrpages(hpa, size);
+ ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
+ hpa >> VTD_PAGE_SHIFT, size, prot);
return ret;
}
git.openpandora.org / pandora-kernel.git / blobdiff