x86/mm: Add barriers and document switch_mm()-vs-flush synchronization

[pandora-kernel.git] / arch / x86 / mm / tlb.c

diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index d6c0418..55034a1 100644 (file)
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -278,7 +278,9 @@ void flush_tlb_current_task(void)
 
        preempt_disable();
 
+       /* This is an implicit full barrier that synchronizes with switch_mm. */
        local_flush_tlb();
+
        if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
                flush_tlb_others(mm_cpumask(mm), mm, TLB_FLUSH_ALL);
        preempt_enable();
@@ -289,10 +291,20 @@ void flush_tlb_mm(struct mm_struct *mm)
        preempt_disable();
 
        if (current->active_mm == mm) {
-               if (current->mm)
+               if (current->mm) {
+                       /*
+                        * This is an implicit full barrier (MOV to CR) that
+                        * synchronizes with switch_mm.
+                        */
                        local_flush_tlb();
-               else
+               } else {
                        leave_mm(smp_processor_id());
+                       /* Synchronize with switch_mm. */
+                       smp_mb();
+               }
+       } else {
+               /* Synchronize with switch_mm. */
+               smp_mb();
        }
        if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
                flush_tlb_others(mm_cpumask(mm), mm, TLB_FLUSH_ALL);
@@ -307,10 +319,18 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
        preempt_disable();
 
        if (current->active_mm == mm) {
-               if (current->mm)
+               if (current->mm) {
+                       /*
+                        * Implicit full barrier (INVLPG) that synchronizes
+                        * with switch_mm.
+                        */
                        __flush_tlb_one(va);
-               else
+               } else {
                        leave_mm(smp_processor_id());
+
+                       /* Synchronize with switch_mm. */
+                       smp_mb();
+               }
        }
 
        if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)