domain->iommu_coherency = 1;
- i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
- for (; i < g_num_of_iommus; ) {
+ for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
if (!ecap_coherent(g_iommus[i]->ecap)) {
domain->iommu_coherency = 0;
break;
}
- i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
}
}
domain->iommu_snooping = 1;
- i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
- for (; i < g_num_of_iommus; ) {
+ for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
if (!ecap_sc_support(g_iommus[i]->ecap)) {
domain->iommu_snooping = 0;
break;
}
- i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
}
}
}
static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
- unsigned long pfn, unsigned int pages)
+ unsigned long pfn, unsigned int pages, int map)
{
unsigned int mask = ilog2(__roundup_pow_of_two(pages));
uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
DMA_TLB_PSI_FLUSH);
/*
- * In caching mode, domain ID 0 is reserved for non-present to present
- * mapping flush. Device IOTLB doesn't need to be flushed in this case.
+ * In caching mode, changes of pages from non-present to present require
+ * flush. However, device IOTLB doesn't need to be flushed in this case.
*/
- if (!cap_caching_mode(iommu->cap) || did)
+ if (!cap_caching_mode(iommu->cap) || !map)
iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
}
unsigned long nlongs;
ndomains = cap_ndoms(iommu->cap);
- pr_debug("Number of Domains supportd <%ld>\n", ndomains);
+ pr_debug("IOMMU %d: Number of Domains supportd <%ld>\n", iommu->seq_id,
+ ndomains);
nlongs = BITS_TO_LONGS(ndomains);
spin_lock_init(&iommu->lock);
unsigned long flags;
if ((iommu->domains) && (iommu->domain_ids)) {
- i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
- for (; i < cap_ndoms(iommu->cap); ) {
+ for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
domain = iommu->domains[i];
clear_bit(i, iommu->domain_ids);
domain_exit(domain);
}
spin_unlock_irqrestore(&domain->iommu_lock, flags);
-
- i = find_next_bit(iommu->domain_ids,
- cap_ndoms(iommu->cap), i+1);
}
}
spin_lock_irqsave(&iommu->lock, flags);
ndomains = cap_ndoms(iommu->cap);
- num = find_first_bit(iommu->domain_ids, ndomains);
- for (; num < ndomains; ) {
+ for_each_set_bit(num, iommu->domain_ids, ndomains) {
if (iommu->domains[num] == domain) {
found = 1;
break;
}
- num = find_next_bit(iommu->domain_ids,
- cap_ndoms(iommu->cap), num+1);
}
if (found) {
/* find an available domain id for this device in iommu */
ndomains = cap_ndoms(iommu->cap);
- num = find_first_bit(iommu->domain_ids, ndomains);
- for (; num < ndomains; ) {
+ for_each_set_bit(num, iommu->domain_ids, ndomains) {
if (iommu->domains[num] == domain) {
id = num;
found = 1;
break;
}
- num = find_next_bit(iommu->domain_ids,
- cap_ndoms(iommu->cap), num+1);
}
if (found == 0) {
(((u16)bus) << 8) | devfn,
DMA_CCMD_MASK_NOBIT,
DMA_CCMD_DEVICE_INVL);
- iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_DSI_FLUSH);
+ iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
} else {
iommu_flush_write_buffer(iommu);
}
*/
iommu->flush.flush_context = __iommu_flush_context;
iommu->flush.flush_iotlb = __iommu_flush_iotlb;
- printk(KERN_INFO "IOMMU 0x%Lx: using Register based "
+ printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
"invalidation\n",
+ iommu->seq_id,
(unsigned long long)drhd->reg_base_addr);
} else {
iommu->flush.flush_context = qi_flush_context;
iommu->flush.flush_iotlb = qi_flush_iotlb;
- printk(KERN_INFO "IOMMU 0x%Lx: using Queued "
+ printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
"invalidation\n",
+ iommu->seq_id,
(unsigned long long)drhd->reg_base_addr);
}
}
/* it's a non-present to present mapping. Only flush if caching mode */
if (cap_caching_mode(iommu->cap))
- iommu_flush_iotlb_psi(iommu, 0, mm_to_dma_pfn(iova->pfn_lo), size);
+ iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 1);
else
iommu_flush_write_buffer(iommu);
if (!deferred_flush[i].next)
continue;
- iommu->flush.flush_iotlb(iommu, 0, 0, 0,
+ /* In caching mode, global flushes turn emulation expensive */
+ if (!cap_caching_mode(iommu->cap))
+ iommu->flush.flush_iotlb(iommu, 0, 0, 0,
DMA_TLB_GLOBAL_FLUSH);
for (j = 0; j < deferred_flush[i].next; j++) {
unsigned long mask;
struct iova *iova = deferred_flush[i].iova[j];
-
- mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
- iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
- (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
+ struct dmar_domain *domain = deferred_flush[i].domain[j];
+
+ /* On real hardware multiple invalidations are expensive */
+ if (cap_caching_mode(iommu->cap))
+ iommu_flush_iotlb_psi(iommu, domain->id,
+ iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 0);
+ else {
+ mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
+ iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
+ (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
+ }
__free_iova(&deferred_flush[i].domain[j]->iovad, iova);
}
deferred_flush[i].next = 0;
if (intel_iommu_strict) {
iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
- last_pfn - start_pfn + 1);
+ last_pfn - start_pfn + 1, 0);
/* free iova */
__free_iova(&domain->iovad, iova);
} else {
if (intel_iommu_strict) {
iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
- last_pfn - start_pfn + 1);
+ last_pfn - start_pfn + 1, 0);
/* free iova */
__free_iova(&domain->iovad, iova);
} else {
struct dmar_domain *domain;
size_t size = 0;
int prot = 0;
- size_t offset_pfn = 0;
struct iova *iova = NULL;
int ret;
struct scatterlist *sg;
/* it's a non-present to present mapping. Only flush if caching mode */
if (cap_caching_mode(iommu->cap))
- iommu_flush_iotlb_psi(iommu, 0, start_vpfn, offset_pfn);
+ iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 1);
else
iommu_flush_write_buffer(iommu);
/* domain id for virtual machine, it won't be set in context */
static unsigned long vm_domid;
-static int vm_domain_min_agaw(struct dmar_domain *domain)
-{
- int i;
- int min_agaw = domain->agaw;
-
- i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
- for (; i < g_num_of_iommus; ) {
- if (min_agaw > g_iommus[i]->agaw)
- min_agaw = g_iommus[i]->agaw;
-
- i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
- }
-
- return min_agaw;
-}
-
static struct dmar_domain *iommu_alloc_vm_domain(void)
{
struct dmar_domain *domain;
iommu = drhd->iommu;
ndomains = cap_ndoms(iommu->cap);
- i = find_first_bit(iommu->domain_ids, ndomains);
- for (; i < ndomains; ) {
+ for_each_set_bit(i, iommu->domain_ids, ndomains) {
if (iommu->domains[i] == domain) {
spin_lock_irqsave(&iommu->lock, flags);
clear_bit(i, iommu->domain_ids);
spin_unlock_irqrestore(&iommu->lock, flags);
break;
}
- i = find_next_bit(iommu->domain_ids, ndomains, i+1);
}
}
}
struct pci_dev *pdev = to_pci_dev(dev);
struct intel_iommu *iommu;
int addr_width;
- u64 end;
/* normally pdev is not mapped */
if (unlikely(domain_context_mapped(pdev))) {
/* check if this iommu agaw is sufficient for max mapped address */
addr_width = agaw_to_width(iommu->agaw);
- end = DOMAIN_MAX_ADDR(addr_width);
- end = end & VTD_PAGE_MASK;
- if (end < dmar_domain->max_addr) {
- printk(KERN_ERR "%s: iommu agaw (%d) is not "
+ if (addr_width > cap_mgaw(iommu->cap))
+ addr_width = cap_mgaw(iommu->cap);
+
+ if (dmar_domain->max_addr > (1LL << addr_width)) {
+ printk(KERN_ERR "%s: iommu width (%d) is not "
"sufficient for the mapped address (%llx)\n",
- __func__, iommu->agaw, dmar_domain->max_addr);
+ __func__, addr_width, dmar_domain->max_addr);
return -EFAULT;
}
+ dmar_domain->gaw = addr_width;
+
+ /*
+ * Knock out extra levels of page tables if necessary
+ */
+ while (iommu->agaw < dmar_domain->agaw) {
+ struct dma_pte *pte;
+
+ pte = dmar_domain->pgd;
+ if (dma_pte_present(pte)) {
+ free_pgtable_page(dmar_domain->pgd);
+ dmar_domain->pgd = (struct dma_pte *)dma_pte_addr(pte);
+ }
+ dmar_domain->agaw--;
+ }
return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
}
{
struct dmar_domain *dmar_domain = domain->priv;
u64 max_addr;
- int addr_width;
int prot = 0;
size_t size;
int ret;
size = PAGE_SIZE << gfp_order;
max_addr = iova + size;
if (dmar_domain->max_addr < max_addr) {
- int min_agaw;
u64 end;
/* check if minimum agaw is sufficient for mapped address */
- min_agaw = vm_domain_min_agaw(dmar_domain);
- addr_width = agaw_to_width(min_agaw);
- end = DOMAIN_MAX_ADDR(addr_width);
- end = end & VTD_PAGE_MASK;
+ end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
if (end < max_addr) {
- printk(KERN_ERR "%s: iommu agaw (%d) is not "
+ printk(KERN_ERR "%s: iommu width (%d) is not "
"sufficient for the mapped address (%llx)\n",
- __func__, min_agaw, max_addr);
+ __func__, dmar_domain->gaw, max_addr);
return -EFAULT;
}
dmar_domain->max_addr = max_addr;
git.openpandora.org / pandora-kernel.git / blobdiff