Merge branches 'slab/fixes', 'slab/kmemleak', 'slub/perf' and 'slub/stats' into for...

[pandora-kernel.git] / mm / slab.c

diff --git a/mm/slab.c b/mm/slab.c
index 66e9047..a6c9166 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -604,6 +604,26 @@ static struct kmem_cache cache_cache = {
 
 #define BAD_ALIEN_MAGIC 0x01020304ul
 
+/*
+ * chicken and egg problem: delay the per-cpu array allocation
+ * until the general caches are up.
+ */
+static enum {
+       NONE,
+       PARTIAL_AC,
+       PARTIAL_L3,
+       EARLY,
+       FULL
+} g_cpucache_up;
+
+/*
+ * used by boot code to determine if it can use slab based allocator
+ */
+int slab_is_available(void)
+{
+       return g_cpucache_up >= EARLY;
+}
+
 #ifdef CONFIG_LOCKDEP
 
 /*
@@ -620,40 +640,52 @@ static struct kmem_cache cache_cache = {
 static struct lock_class_key on_slab_l3_key;
 static struct lock_class_key on_slab_alc_key;
 
-static inline void init_lock_keys(void)
-
+static void init_node_lock_keys(int q)
 {
-       int q;
        struct cache_sizes *s = malloc_sizes;
 
-       while (s->cs_size != ULONG_MAX) {
-               for_each_node(q) {
-                       struct array_cache **alc;
-                       int r;
-                       struct kmem_list3 *l3 = s->cs_cachep->nodelists[q];
-                       if (!l3 || OFF_SLAB(s->cs_cachep))
-                               continue;
-                       lockdep_set_class(&l3->list_lock, &on_slab_l3_key);
-                       alc = l3->alien;
-                       /*
-                        * FIXME: This check for BAD_ALIEN_MAGIC
-                        * should go away when common slab code is taught to
-                        * work even without alien caches.
-                        * Currently, non NUMA code returns BAD_ALIEN_MAGIC
-                        * for alloc_alien_cache,
-                        */
-                       if (!alc || (unsigned long)alc == BAD_ALIEN_MAGIC)
-                               continue;
-                       for_each_node(r) {
-                               if (alc[r])
-                                       lockdep_set_class(&alc[r]->lock,
-                                            &on_slab_alc_key);
-                       }
+       if (g_cpucache_up != FULL)
+               return;
+
+       for (s = malloc_sizes; s->cs_size != ULONG_MAX; s++) {
+               struct array_cache **alc;
+               struct kmem_list3 *l3;
+               int r;
+
+               l3 = s->cs_cachep->nodelists[q];
+               if (!l3 || OFF_SLAB(s->cs_cachep))
+                       return;
+               lockdep_set_class(&l3->list_lock, &on_slab_l3_key);
+               alc = l3->alien;
+               /*
+                * FIXME: This check for BAD_ALIEN_MAGIC
+                * should go away when common slab code is taught to
+                * work even without alien caches.
+                * Currently, non NUMA code returns BAD_ALIEN_MAGIC
+                * for alloc_alien_cache,
+                */
+               if (!alc || (unsigned long)alc == BAD_ALIEN_MAGIC)
+                       return;
+               for_each_node(r) {
+                       if (alc[r])
+                               lockdep_set_class(&alc[r]->lock,
+                                       &on_slab_alc_key);
                }
-               s++;
        }
 }
+
+static inline void init_lock_keys(void)
+{
+       int node;
+
+       for_each_node(node)
+               init_node_lock_keys(node);
+}
 #else
+static void init_node_lock_keys(int q)
+{
+}
+
 static inline void init_lock_keys(void)
 {
 }
@@ -665,26 +697,6 @@ static inline void init_lock_keys(void)
 static DEFINE_MUTEX(cache_chain_mutex);
 static struct list_head cache_chain;
 
-/*
- * chicken and egg problem: delay the per-cpu array allocation
- * until the general caches are up.
- */
-static enum {
-       NONE,
-       PARTIAL_AC,
-       PARTIAL_L3,
-       EARLY,
-       FULL
-} g_cpucache_up;
-
-/*
- * used by boot code to determine if it can use slab based allocator
- */
-int slab_is_available(void)
-{
-       return g_cpucache_up >= EARLY;
-}
-
 static DEFINE_PER_CPU(struct delayed_work, reap_work);
 
 static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep)
@@ -1254,6 +1266,8 @@ static int __cpuinit cpuup_prepare(long cpu)
                kfree(shared);
                free_alien_cache(alien);
        }
+       init_node_lock_keys(node);
+
        return 0;
 bad:
        cpuup_canceled(cpu);
@@ -3312,7 +3326,7 @@ __cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
        cache_alloc_debugcheck_before(cachep, flags);
        local_irq_save(save_flags);
 
-       if (unlikely(nodeid == -1))
+       if (nodeid == -1)
                nodeid = numa_node_id();
 
        if (unlikely(!cachep->nodelists[nodeid])) {