arm64: mm: Optimise tlb flush logic where we have >4K granule

[pandora-kernel.git] / arch / arm64 / include / asm / tlbflush.h

diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h
index 8b48203..7881d7d 100644 (file)
--- a/arch/arm64/include/asm/tlbflush.h
+++ b/arch/arm64/include/asm/tlbflush.h
@@ -98,11 +98,31 @@ static inline void flush_tlb_page(struct vm_area_struct *vma,
        dsb();
 }
 
-/*
- * Convert calls to our calling convention.
- */
-#define flush_tlb_range(vma,start,end) __cpu_flush_user_tlb_range(start,end,vma)
-#define flush_tlb_kernel_range(s,e)    __cpu_flush_kern_tlb_range(s,e)
+static inline void flush_tlb_range(struct vm_area_struct *vma,
+                                       unsigned long start, unsigned long end)
+{
+       unsigned long asid = (unsigned long)ASID(vma->vm_mm) << 48;
+       unsigned long addr;
+       start = asid | (start >> 12);
+       end = asid | (end >> 12);
+
+       dsb(ishst);
+       for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12))
+               asm("tlbi vae1is, %0" : : "r"(addr));
+       dsb(ish);
+}
+
+static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)
+{
+       unsigned long addr;
+       start >>= 12;
+       end >>= 12;
+
+       dsb(ishst);
+       for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12))
+               asm("tlbi vaae1is, %0" : : "r"(addr));
+       dsb(ish);
+}
 
 /*
  * On AArch64, the cache coherency is handled via the set_pte_at() function.