[MIPS] Fix aliasing bug in copy_to_user_page / copy_from_user_page

The current implementation uses a sequence of a cacheflush and a copy.
This is racy in case of a multithreaded debuggee and renders GDB
virtually unusable.

Aside this fixes a performance hog rendering access to /proc/cmdline very
slow and resulting in a enough cache stalls for the 34K AP/SP programming
model to make the bare metal code on the non-Linux VPE miss RT deadlines.

The main part of this patch was originally written by Ralf Baechle;
Atushi Nemoto did the the debugging.

Signed-off-by: Atsushi Nemoto <anemo@mba.ocn.ne.jp>
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>

diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c
index 88b72c9..2de4d3c 100644 (file)
--- a/arch/mips/mm/init.c
+++ b/arch/mips/mm/init.c
@@ -30,11 +30,34 @@
 #include <asm/cachectl.h>
 #include <asm/cpu.h>
 #include <asm/dma.h>
+#include <asm/kmap_types.h>
 #include <asm/mmu_context.h>
 #include <asm/sections.h>
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
 #include <asm/tlb.h>
+#include <asm/fixmap.h>
+
+/* Atomicity and interruptability */
+#ifdef CONFIG_MIPS_MT_SMTC
+
+#include <asm/mipsmtregs.h>
+
+#define ENTER_CRITICAL(flags) \
+       { \
+       unsigned int mvpflags; \
+       local_irq_save(flags);\
+       mvpflags = dvpe()
+#define EXIT_CRITICAL(flags) \
+       evpe(mvpflags); \
+       local_irq_restore(flags); \
+       }
+#else
+
+#define ENTER_CRITICAL(flags) local_irq_save(flags)
+#define EXIT_CRITICAL(flags) local_irq_restore(flags)
+
+#endif /* CONFIG_MIPS_MT_SMTC */
 
 DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
 
@@ -80,13 +103,142 @@ unsigned long setup_zero_pages(void)
        return 1UL << order;
 }
 
-#ifdef CONFIG_HIGHMEM
-pte_t *kmap_pte;
-pgprot_t kmap_prot;
+/*
+ * These are almost like kmap_atomic / kunmap_atmic except they take an
+ * additional address argument as the hint.
+ */
 
 #define kmap_get_fixmap_pte(vaddr)                                     \
        pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr)), (vaddr))
 
+#ifdef CONFIG_MIPS_MT_SMTC
+static pte_t *kmap_coherent_pte;
+static void __init kmap_coherent_init(void)
+{
+       unsigned long vaddr;
+
+       /* cache the first coherent kmap pte */
+       vaddr = __fix_to_virt(FIX_CMAP_BEGIN);
+       kmap_coherent_pte = kmap_get_fixmap_pte(vaddr);
+}
+#else
+static inline void kmap_coherent_init(void) {}
+#endif
+
+static inline void *kmap_coherent(struct page *page, unsigned long addr)
+{
+       enum fixed_addresses idx;
+       unsigned long vaddr, flags, entrylo;
+       unsigned long old_ctx;
+       pte_t pte;
+       int tlbidx;
+
+       inc_preempt_count();
+       idx = (addr >> PAGE_SHIFT) & (FIX_N_COLOURS - 1);
+#ifdef CONFIG_MIPS_MT_SMTC
+       idx += FIX_N_COLOURS * smp_processor_id();
+#endif
+       vaddr = __fix_to_virt(FIX_CMAP_END - idx);
+       pte = mk_pte(page, PAGE_KERNEL);
+#if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32_R1)
+       entrylo = pte.pte_high;
+#else
+       entrylo = pte_val(pte) >> 6;
+#endif
+
+       ENTER_CRITICAL(flags);
+       old_ctx = read_c0_entryhi();
+       write_c0_entryhi(vaddr & (PAGE_MASK << 1));
+       write_c0_entrylo0(entrylo);
+       write_c0_entrylo1(entrylo);
+#ifdef CONFIG_MIPS_MT_SMTC
+       set_pte(kmap_coherent_pte - (FIX_CMAP_END - idx), pte);
+       /* preload TLB instead of local_flush_tlb_one() */
+       mtc0_tlbw_hazard();
+       tlb_probe();
+       tlb_probe_hazard();
+       tlbidx = read_c0_index();
+       mtc0_tlbw_hazard();
+       if (tlbidx < 0)
+               tlb_write_random();
+       else
+               tlb_write_indexed();
+#else
+       tlbidx = read_c0_wired();
+       write_c0_wired(tlbidx + 1);
+       write_c0_index(tlbidx);
+       mtc0_tlbw_hazard();
+       tlb_write_indexed();
+#endif
+       tlbw_use_hazard();
+       write_c0_entryhi(old_ctx);
+       EXIT_CRITICAL(flags);
+
+       return (void*) vaddr;
+}
+
+#define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1)))
+
+static inline void kunmap_coherent(struct page *page)
+{
+#ifndef CONFIG_MIPS_MT_SMTC
+       unsigned int wired;
+       unsigned long flags, old_ctx;
+
+       ENTER_CRITICAL(flags);
+       old_ctx = read_c0_entryhi();
+       wired = read_c0_wired() - 1;
+       write_c0_wired(wired);
+       write_c0_index(wired);
+       write_c0_entryhi(UNIQUE_ENTRYHI(wired));
+       write_c0_entrylo0(0);
+       write_c0_entrylo1(0);
+       mtc0_tlbw_hazard();
+       tlb_write_indexed();
+       tlbw_use_hazard();
+       write_c0_entryhi(old_ctx);
+       EXIT_CRITICAL(flags);
+#endif
+       dec_preempt_count();
+       preempt_check_resched();
+}
+
+void copy_to_user_page(struct vm_area_struct *vma,
+       struct page *page, unsigned long vaddr, void *dst, const void *src,
+       unsigned long len)
+{
+       if (cpu_has_dc_aliases) {
+               void *vto = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
+               memcpy(vto, src, len);
+               kunmap_coherent(page);
+       } else
+               memcpy(dst, src, len);
+       if ((vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc)
+               flush_cache_page(vma, vaddr, page_to_pfn(page));
+}
+
+EXPORT_SYMBOL(copy_to_user_page);
+
+void copy_from_user_page(struct vm_area_struct *vma,
+       struct page *page, unsigned long vaddr, void *dst, const void *src,
+       unsigned long len)
+{
+       if (cpu_has_dc_aliases) {
+               void *vfrom =
+                       kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
+               memcpy(dst, vfrom, len);
+               kunmap_coherent(page);
+       } else
+               memcpy(dst, src, len);
+}
+
+EXPORT_SYMBOL(copy_from_user_page);
+
+
+#ifdef CONFIG_HIGHMEM
+pte_t *kmap_pte;
+pgprot_t kmap_prot;
+
 static void __init kmap_init(void)
 {
        unsigned long kmap_vstart;
@@ -97,11 +249,12 @@ static void __init kmap_init(void)
 
        kmap_prot = PAGE_KERNEL;
 }
+#endif /* CONFIG_HIGHMEM */
 
-#ifdef CONFIG_32BIT
 void __init fixrange_init(unsigned long start, unsigned long end,
        pgd_t *pgd_base)
 {
+#if defined(CONFIG_HIGHMEM) || defined(CONFIG_MIPS_MT_SMTC)
        pgd_t *pgd;
        pud_t *pud;
        pmd_t *pmd;
@@ -122,7 +275,7 @@ void __init fixrange_init(unsigned long start, unsigned long end,
                        for (; (k < PTRS_PER_PMD) && (vaddr != end); pmd++, k++) {
                                if (pmd_none(*pmd)) {
                                        pte = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE);
-                                       set_pmd(pmd, __pmd(pte));
+                                       set_pmd(pmd, __pmd((unsigned long)pte));
                                        if (pte != pte_offset_kernel(pmd, 0))
                                                BUG();
                                }
@@ -132,9 +285,8 @@ void __init fixrange_init(unsigned long start, unsigned long end,
                }
                j = 0;
        }
+#endif
 }
-#endif /* CONFIG_32BIT */
-#endif /* CONFIG_HIGHMEM */
 
 #ifndef CONFIG_NEED_MULTIPLE_NODES
 extern void pagetable_init(void);
@@ -175,6 +327,7 @@ void __init paging_init(void)
 #ifdef CONFIG_HIGHMEM
        kmap_init();
 #endif
+       kmap_coherent_init();
 
        max_dma = virt_to_phys((char *)MAX_DMA_ADDRESS) >> PAGE_SHIFT;
        low = max_low_pfn;

diff --git a/arch/mips/mm/pgtable-32.c b/arch/mips/mm/pgtable-32.c
index 4bdaa05..4a61e62 100644 (file)
--- a/arch/mips/mm/pgtable-32.c
+++ b/arch/mips/mm/pgtable-32.c
@@ -31,9 +31,10 @@ void pgd_init(unsigned long page)
 
 void __init pagetable_init(void)
 {
-#ifdef CONFIG_HIGHMEM
        unsigned long vaddr;
-       pgd_t *pgd, *pgd_base;
+       pgd_t *pgd_base;
+#ifdef CONFIG_HIGHMEM
+       pgd_t *pgd;
        pud_t *pud;
        pmd_t *pmd;
        pte_t *pte;
@@ -44,7 +45,6 @@ void __init pagetable_init(void)
        pgd_init((unsigned long)swapper_pg_dir
                 + sizeof(pgd_t) * USER_PTRS_PER_PGD);
 
-#ifdef CONFIG_HIGHMEM
        pgd_base = swapper_pg_dir;
 
        /*
@@ -53,6 +53,7 @@ void __init pagetable_init(void)
        vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;
        fixrange_init(vaddr, 0, pgd_base);
 
+#ifdef CONFIG_HIGHMEM
        /*
         * Permanent kmaps:
         */

diff --git a/arch/mips/mm/pgtable-64.c b/arch/mips/mm/pgtable-64.c
index 44b5e97..8d600d3 100644 (file)
--- a/arch/mips/mm/pgtable-64.c
+++ b/arch/mips/mm/pgtable-64.c
@@ -8,6 +8,7 @@
  */
 #include <linux/init.h>
 #include <linux/mm.h>
+#include <asm/fixmap.h>
 #include <asm/pgtable.h>
 
 void pgd_init(unsigned long page)
@@ -52,7 +53,17 @@ void pmd_init(unsigned long addr, unsigned long pagetable)
 
 void __init pagetable_init(void)
 {
+       unsigned long vaddr;
+       pgd_t *pgd_base;
+
        /* Initialize the entire pgd.  */
        pgd_init((unsigned long)swapper_pg_dir);
        pmd_init((unsigned long)invalid_pmd_table, (unsigned long)invalid_pte_table);
+
+       pgd_base = swapper_pg_dir;
+       /*
+        * Fixed mappings:
+        */
+       vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;
+       fixrange_init(vaddr, 0, pgd_base);
 }

diff --git a/include/asm-mips/cacheflush.h b/include/asm-mips/cacheflush.h
index 9ab59e2..e3c9925 100644 (file)
--- a/include/asm-mips/cacheflush.h
+++ b/include/asm-mips/cacheflush.h
@@ -55,24 +55,13 @@ extern void (*flush_icache_range)(unsigned long start, unsigned long end);
 #define flush_cache_vmap(start, end)           flush_cache_all()
 #define flush_cache_vunmap(start, end)         flush_cache_all()
 
-static inline void copy_to_user_page(struct vm_area_struct *vma,
+extern void copy_to_user_page(struct vm_area_struct *vma,
        struct page *page, unsigned long vaddr, void *dst, const void *src,
-       unsigned long len)
-{
-       if (cpu_has_dc_aliases)
-               flush_cache_page(vma, vaddr, page_to_pfn(page));
-       memcpy(dst, src, len);
-       __flush_icache_page(vma, page);
-}
+       unsigned long len);
 
-static inline void copy_from_user_page(struct vm_area_struct *vma,
+extern void copy_from_user_page(struct vm_area_struct *vma,
        struct page *page, unsigned long vaddr, void *dst, const void *src,
-       unsigned long len)
-{
-       if (cpu_has_dc_aliases)
-               flush_cache_page(vma, vaddr, page_to_pfn(page));
-       memcpy(dst, src, len);
-}
+       unsigned long len);
 
 extern void (*flush_cache_sigtramp)(unsigned long addr);
 extern void (*flush_icache_all)(void);

diff --git a/include/asm-mips/fixmap.h b/include/asm-mips/fixmap.h
index 6959bdb..02c8a13 100644 (file)
--- a/include/asm-mips/fixmap.h
+++ b/include/asm-mips/fixmap.h
@@ -45,8 +45,16 @@
  * fix-mapped?
  */
 enum fixed_addresses {
+#define FIX_N_COLOURS 8
+       FIX_CMAP_BEGIN,
+#ifdef CONFIG_MIPS_MT_SMTC
+       FIX_CMAP_END = FIX_CMAP_BEGIN + (FIX_N_COLOURS * NR_CPUS),
+#else
+       FIX_CMAP_END = FIX_CMAP_BEGIN + FIX_N_COLOURS,
+#endif
 #ifdef CONFIG_HIGHMEM
-       FIX_KMAP_BEGIN, /* reserved pte's for temporary kernel mappings */
+       /* reserved pte's for temporary kernel mappings */
+       FIX_KMAP_BEGIN = FIX_CMAP_END + 1,
        FIX_KMAP_END = FIX_KMAP_BEGIN+(KM_TYPE_NR*NR_CPUS)-1,
 #endif
        __end_of_fixed_addresses
@@ -70,9 +78,9 @@ extern void __set_fixmap (enum fixed_addresses idx,
  * at the top of mem..
  */
 #if defined(CONFIG_CPU_TX39XX) || defined(CONFIG_CPU_TX49XX)
-#define FIXADDR_TOP    (0xff000000UL - 0x2000)
+#define FIXADDR_TOP    ((unsigned long)(long)(int)(0xff000000 - 0x20000))
 #else
-#define FIXADDR_TOP    (0xffffe000UL)
+#define FIXADDR_TOP    ((unsigned long)(long)(int)0xfffe0000)
 #endif
 #define FIXADDR_SIZE   (__end_of_fixed_addresses << PAGE_SHIFT)
 #define FIXADDR_START  (FIXADDR_TOP - FIXADDR_SIZE)