mm: convert mm->cpu_vm_cpumask into cpumask_var_t

cpumask_t is very big struct and cpu_vm_mask is placed wrong position.
It might lead to reduce cache hit ratio.

This patch has two change.
1) Move the place of cpumask into last of mm_struct. Because usually cpumask
   is accessed only front bits when the system has cpu-hotplug capability
2) Convert cpu_vm_mask into cpumask_var_t. It may help to reduce memory
   footprint if cpumask_size() will use nr_cpumask_bits properly in future.

In addition, this patch change the name of cpu_vm_mask with cpu_vm_mask_var.
It may help to detect out of tree cpu_vm_mask users.

This patch has no functional change.

[akpm@linux-foundation.org: build fix]
[akpm@linux-foundation.org: coding-style fixes]
Signed-off-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Koichi Yasutake <yasutake.koichi@jp.panasonic.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Chris Metcalf <cmetcalf@tilera.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/Documentation/cachetlb.txt b/Documentation/cachetlb.txt
index 9164ae3..9b728dc 100644 (file)
--- a/Documentation/cachetlb.txt
+++ b/Documentation/cachetlb.txt
@@ -16,7 +16,7 @@ on all processors in the system.  Don't let this scare you into
 thinking SMP cache/tlb flushing must be so inefficient, this is in
 fact an area where many optimizations are possible.  For example,
 if it can be proven that a user address space has never executed
-on a cpu (see vma->cpu_vm_mask), one need not perform a flush
+on a cpu (see mm_cpumask()), one need not perform a flush
 for this address space on that cpu.
 
 First, the TLB flushing interfaces, since they are the simplest.  The

diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c
index 998e972..30ac65d 100644 (file)
--- a/arch/x86/kernel/tboot.c
+++ b/arch/x86/kernel/tboot.c
@@ -110,7 +110,6 @@ static struct mm_struct tboot_mm = {
        .mmap_sem       = __RWSEM_INITIALIZER(init_mm.mmap_sem),
        .page_table_lock =  __SPIN_LOCK_UNLOCKED(init_mm.page_table_lock),
        .mmlist         = LIST_HEAD_INIT(init_mm.mmlist),
-       .cpu_vm_mask    = CPU_MASK_ALL,
 };
 
 static inline void switch_to_tboot_pt(void)

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 201998e..c2f9ea7 100644 (file)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -265,8 +265,6 @@ struct mm_struct {
 
        struct linux_binfmt *binfmt;
 
-       cpumask_t cpu_vm_mask;
-
        /* Architecture-specific MM context */
        mm_context_t context;
 
@@ -316,9 +314,14 @@ struct mm_struct {
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
        pgtable_t pmd_huge_pte; /* protected by page_table_lock */
 #endif
+
+       cpumask_var_t cpu_vm_mask_var;
 };
 
 /* Future-safe accessor for struct mm_struct's cpu_vm_mask. */
-#define mm_cpumask(mm) (&(mm)->cpu_vm_mask)
+static inline cpumask_t *mm_cpumask(struct mm_struct *mm)
+{
+       return mm->cpu_vm_mask_var;
+}
 
 #endif /* _LINUX_MM_TYPES_H */

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 44b8faa..f18300e 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2176,6 +2176,7 @@ static inline void mmdrop(struct mm_struct * mm)
        if (unlikely(atomic_dec_and_test(&mm->mm_count)))
                __mmdrop(mm);
 }
+extern int mm_init_cpumask(struct mm_struct *mm, struct mm_struct *oldmm);
 
 /* mmput gets rid of the mappings and all user-space */
 extern void mmput(struct mm_struct *);

diff --git a/init/main.c b/init/main.c
index 48df882..22da339 100644 (file)
--- a/init/main.c
+++ b/init/main.c
@@ -509,6 +509,8 @@ asmlinkage void __init start_kernel(void)
        sort_main_extable();
        trap_init();
        mm_init();
+       BUG_ON(mm_init_cpumask(&init_mm, 0));
+
        /*
         * Set up the scheduler prior starting any interrupts (such as the
         * timer interrupt). Full topology setup happens at smp_init()

diff --git a/kernel/fork.c b/kernel/fork.c
index 9276927..8e7e135 100644 (file)
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -485,6 +485,20 @@ static void mm_init_aio(struct mm_struct *mm)
 #endif
 }
 
+int mm_init_cpumask(struct mm_struct *mm, struct mm_struct *oldmm)
+{
+#ifdef CONFIG_CPUMASK_OFFSTACK
+       if (!alloc_cpumask_var(&mm->cpu_vm_mask_var, GFP_KERNEL))
+               return -ENOMEM;
+
+       if (oldmm)
+               cpumask_copy(mm_cpumask(mm), mm_cpumask(oldmm));
+       else
+               memset(mm_cpumask(mm), 0, cpumask_size());
+#endif
+       return 0;
+}
+
 static struct mm_struct * mm_init(struct mm_struct * mm, struct task_struct *p)
 {
        atomic_set(&mm->mm_users, 1);
@@ -521,10 +535,20 @@ struct mm_struct * mm_alloc(void)
        struct mm_struct * mm;
 
        mm = allocate_mm();
-       if (mm) {
-               memset(mm, 0, sizeof(*mm));
-               mm = mm_init(mm, current);
+       if (!mm)
+               return NULL;
+
+       memset(mm, 0, sizeof(*mm));
+       mm = mm_init(mm, current);
+       if (!mm)
+               return NULL;
+
+       if (mm_init_cpumask(mm, NULL)) {
+               mm_free_pgd(mm);
+               free_mm(mm);
+               return NULL;
        }
+
        return mm;
 }
 
@@ -536,6 +560,7 @@ struct mm_struct * mm_alloc(void)
 void __mmdrop(struct mm_struct *mm)
 {
        BUG_ON(mm == &init_mm);
+       free_cpumask_var(mm->cpu_vm_mask_var);
        mm_free_pgd(mm);
        destroy_context(mm);
        mmu_notifier_mm_destroy(mm);
@@ -690,6 +715,9 @@ struct mm_struct *dup_mm(struct task_struct *tsk)
        if (!mm_init(mm, tsk))
                goto fail_nomem;
 
+       if (mm_init_cpumask(mm, oldmm))
+               goto fail_nocpumask;
+
        if (init_new_context(tsk, mm))
                goto fail_nocontext;
 
@@ -716,6 +744,9 @@ fail_nomem:
        return NULL;
 
 fail_nocontext:
+       free_cpumask_var(mm->cpu_vm_mask_var);
+
+fail_nocpumask:
        /*
         * If init_new_context() failed, we cannot use mmput() to free the mm
         * because it calls destroy_context()

diff --git a/mm/init-mm.c b/mm/init-mm.c
index 1d29cdf..4019979 100644 (file)
--- a/mm/init-mm.c
+++ b/mm/init-mm.c
@@ -21,6 +21,5 @@ struct mm_struct init_mm = {
        .mmap_sem       = __RWSEM_INITIALIZER(init_mm.mmap_sem),
        .page_table_lock =  __SPIN_LOCK_UNLOCKED(init_mm.page_table_lock),
        .mmlist         = LIST_HEAD_INIT(init_mm.mmlist),
-       .cpu_vm_mask    = CPU_MASK_ALL,
        INIT_MM_CONTEXT(init_mm)
 };