Merge git://git.kernel.org/pub/scm/linux/kernel/git/steve/gfs2-2.6-fixes

[pandora-kernel.git] / mm / allocpercpu.c

diff --git a/mm/allocpercpu.c b/mm/allocpercpu.c
index 7e58322..f4026ba 100644 (file)
--- a/mm/allocpercpu.c
+++ b/mm/allocpercpu.c
@@ -6,6 +6,10 @@
 #include <linux/mm.h>
 #include <linux/module.h>
 
+#ifndef cache_line_size
+#define cache_line_size()      L1_CACHE_BYTES
+#endif
+
 /**
  * percpu_depopulate - depopulate per-cpu data for given cpu
  * @__pdata: per-cpu data to depopulate
@@ -52,6 +56,11 @@ void *percpu_populate(void *__pdata, size_t size, gfp_t gfp, int cpu)
        struct percpu_data *pdata = __percpu_disguise(__pdata);
        int node = cpu_to_node(cpu);
 
+       /*
+        * We should make sure each CPU gets private memory.
+        */
+       size = roundup(size, cache_line_size());
+
        BUG_ON(pdata->ptrs[cpu]);
        if (node_online(node))
                pdata->ptrs[cpu] = kmalloc_node(size, gfp|__GFP_ZERO, node);
@@ -73,9 +82,10 @@ EXPORT_SYMBOL_GPL(percpu_populate);
 int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp,
                           cpumask_t *mask)
 {
-       cpumask_t populated = CPU_MASK_NONE;
+       cpumask_t populated;
        int cpu;
 
+       cpus_clear(populated);
        for_each_cpu_mask(cpu, *mask)
                if (unlikely(!percpu_populate(__pdata, size, gfp, cpu))) {
                        __percpu_depopulate_mask(__pdata, &populated);
@@ -98,7 +108,11 @@ EXPORT_SYMBOL_GPL(__percpu_populate_mask);
  */
 void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask)
 {
-       void *pdata = kzalloc(nr_cpu_ids * sizeof(void *), gfp);
+       /*
+        * We allocate whole cache lines to avoid false sharing
+        */
+       size_t sz = roundup(nr_cpu_ids * sizeof(void *), cache_line_size());
+       void *pdata = kzalloc(sz, gfp);
        void *__pdata = __percpu_disguise(pdata);
 
        if (unlikely(!pdata))