if (tmp + level_size(level) - 1 > last_pfn)
return;
- while (tmp <= last_pfn) {
+ while (tmp + level_size(level) - 1 <= last_pfn) {
pte = dma_pfn_level_pte(domain, tmp, level);
if (pte) {
free_pgtable_page(
tmp->devfn);
}
-static int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
- unsigned long phys_pfn, unsigned long nr_pages,
- int prot)
+static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
+ struct scatterlist *sg, unsigned long phys_pfn,
+ unsigned long nr_pages, int prot)
{
struct dma_pte *first_pte = NULL, *pte = NULL;
+ phys_addr_t uninitialized_var(pteval);
int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
+ unsigned long sg_res;
BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
+ if (sg)
+ sg_res = 0;
+ else {
+ sg_res = nr_pages + 1;
+ pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
+ }
+
while (nr_pages--) {
+ if (!sg_res) {
+ sg_res = (sg->offset + sg->length + VTD_PAGE_SIZE - 1) >> VTD_PAGE_SHIFT;
+ sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
+ sg->dma_length = sg->length;
+ pteval = page_to_phys(sg_page(sg)) | prot;
+ }
if (!pte) {
first_pte = pte = pfn_to_dma_pte(domain, iov_pfn);
if (!pte)
/* We don't need lock here, nobody else
* touches the iova range
*/
- BUG_ON(dma_pte_addr(pte));
- pte->val = (phys_pfn << VTD_PAGE_SHIFT) | prot;
+ if (unlikely(dma_pte_addr(pte))) {
+ static int dumps = 5;
+ printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx)\n",
+ iov_pfn, pte->val);
+ if (dumps) {
+ dumps--;
+ debug_dma_dump_mappings(NULL);
+ }
+ WARN_ON(1);
+ }
+ pte->val = pteval;
pte++;
if (!nr_pages ||
(unsigned long)pte >> VTD_PAGE_SHIFT !=
pte = NULL;
}
iov_pfn++;
- phys_pfn++;
+ pteval += VTD_PAGE_SIZE;
+ sg_res--;
+ if (!sg_res)
+ sg = sg_next(sg);
}
return 0;
}
+static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
+ struct scatterlist *sg, unsigned long nr_pages,
+ int prot)
+{
+ return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
+}
+
+static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
+ unsigned long phys_pfn, unsigned long nr_pages,
+ int prot)
+{
+ return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
+}
+
static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
{
if (!iommu)
return host_addr >> VTD_PAGE_SHIFT;
}
-struct iova *
-iommu_alloc_iova(struct dmar_domain *domain, size_t size, u64 end)
-{
- struct iova *piova;
-
- /* Make sure it's in range */
- end = min_t(u64, DOMAIN_MAX_ADDR(domain->gaw), end);
- if (!size || (IOVA_START_ADDR + size > end))
- return NULL;
-
- piova = alloc_iova(&domain->iovad,
- size >> PAGE_SHIFT, IOVA_PFN(end), 1);
- return piova;
-}
-
-static struct iova *
-__intel_alloc_iova(struct device *dev, struct dmar_domain *domain,
- size_t size, u64 dma_mask)
+static struct iova *intel_alloc_iova(struct device *dev,
+ struct dmar_domain *domain,
+ unsigned long nrpages, uint64_t dma_mask)
{
struct pci_dev *pdev = to_pci_dev(dev);
struct iova *iova = NULL;
- if (dma_mask <= DMA_BIT_MASK(32) || dmar_forcedac)
- iova = iommu_alloc_iova(domain, size, dma_mask);
- else {
+ /* Restrict dma_mask to the width that the iommu can handle */
+ dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
+
+ if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
/*
* First try to allocate an io virtual address in
* DMA_BIT_MASK(32) and if that fails then try allocating
* from higher range
*/
- iova = iommu_alloc_iova(domain, size, DMA_BIT_MASK(32));
- if (!iova)
- iova = iommu_alloc_iova(domain, size, dma_mask);
- }
-
- if (!iova) {
- printk(KERN_ERR"Allocating iova for %s failed", pci_name(pdev));
+ iova = alloc_iova(&domain->iovad, nrpages,
+ IOVA_PFN(DMA_BIT_MASK(32)), 1);
+ if (iova)
+ return iova;
+ }
+ iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
+ if (unlikely(!iova)) {
+ printk(KERN_ERR "Allocating %ld-page iova for %s failed",
+ nrpages, pci_name(pdev));
return NULL;
}
iommu = domain_get_iommu(domain);
size = aligned_nrpages(paddr, size);
- iova = __intel_alloc_iova(hwdev, domain, size << VTD_PAGE_SHIFT, pdev->dma_mask);
+ iova = intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
if (!iova)
goto error;
for_each_sg(sglist, sg, nelems, i)
size += aligned_nrpages(sg->offset, sg->length);
- iova = __intel_alloc_iova(hwdev, domain, size << VTD_PAGE_SHIFT,
- pdev->dma_mask);
+ iova = intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
if (!iova) {
sglist->dma_length = 0;
return 0;
prot |= DMA_PTE_WRITE;
start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
- offset_pfn = 0;
- for_each_sg(sglist, sg, nelems, i) {
- int nr_pages = aligned_nrpages(sg->offset, sg->length);
- ret = domain_pfn_mapping(domain, start_vpfn + offset_pfn,
- page_to_dma_pfn(sg_page(sg)),
- nr_pages, prot);
- if (ret) {
- /* clear the page */
- dma_pte_clear_range(domain, start_vpfn,
- start_vpfn + offset_pfn);
- /* free page tables */
- dma_pte_free_pagetable(domain, start_vpfn,
- start_vpfn + offset_pfn);
- /* free iova */
- __free_iova(&domain->iovad, iova);
- return 0;
- }
- sg->dma_address = ((dma_addr_t)(start_vpfn + offset_pfn)
- << VTD_PAGE_SHIFT) + sg->offset;
- sg->dma_length = sg->length;
- offset_pfn += nr_pages;
+
+ ret = domain_sg_mapping(domain, start_vpfn, sglist, mm_to_dma_pfn(size), prot);
+ if (unlikely(ret)) {
+ /* clear the page */
+ dma_pte_clear_range(domain, start_vpfn,
+ start_vpfn + size - 1);
+ /* free page tables */
+ dma_pte_free_pagetable(domain, start_vpfn,
+ start_vpfn + size - 1);
+ /* free iova */
+ __free_iova(&domain->iovad, iova);
+ return 0;
}
/* it's a non-present to present mapping. Only flush if caching mode */
git.openpandora.org / pandora-kernel.git / blobdiff