slab, lockdep: Annotate slab -> rcu -> debug_object -> slab

[pandora-kernel.git] / mm / slab.c

diff --git a/mm/slab.c b/mm/slab.c
index d96e223..0703578 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -574,7 +574,9 @@ static struct arraycache_init initarray_generic =
     { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
 
 /* internal cache of cache description objs */
+static struct kmem_list3 *cache_cache_nodelists[MAX_NUMNODES];
 static struct kmem_cache cache_cache = {
+       .nodelists = cache_cache_nodelists,
        .batchcount = 1,
        .limit = BOOT_CPUCACHE_ENTRIES,
        .shared = 1,
@@ -620,6 +622,51 @@ int slab_is_available(void)
 static struct lock_class_key on_slab_l3_key;
 static struct lock_class_key on_slab_alc_key;
 
+static struct lock_class_key debugobj_l3_key;
+static struct lock_class_key debugobj_alc_key;
+
+static void slab_set_lock_classes(struct kmem_cache *cachep,
+               struct lock_class_key *l3_key, struct lock_class_key *alc_key,
+               int q)
+{
+       struct array_cache **alc;
+       struct kmem_list3 *l3;
+       int r;
+
+       l3 = cachep->nodelists[q];
+       if (!l3)
+               return;
+
+       lockdep_set_class(&l3->list_lock, l3_key);
+       alc = l3->alien;
+       /*
+        * FIXME: This check for BAD_ALIEN_MAGIC
+        * should go away when common slab code is taught to
+        * work even without alien caches.
+        * Currently, non NUMA code returns BAD_ALIEN_MAGIC
+        * for alloc_alien_cache,
+        */
+       if (!alc || (unsigned long)alc == BAD_ALIEN_MAGIC)
+               return;
+       for_each_node(r) {
+               if (alc[r])
+                       lockdep_set_class(&alc[r]->lock, alc_key);
+       }
+}
+
+static void slab_set_debugobj_lock_classes_node(struct kmem_cache *cachep, int node)
+{
+       slab_set_lock_classes(cachep, &debugobj_l3_key, &debugobj_alc_key, node);
+}
+
+static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep)
+{
+       int node;
+
+       for_each_online_node(node)
+               slab_set_debugobj_lock_classes_node(cachep, node);
+}
+
 static void init_node_lock_keys(int q)
 {
        struct cache_sizes *s = malloc_sizes;
@@ -628,29 +675,14 @@ static void init_node_lock_keys(int q)
                return;
 
        for (s = malloc_sizes; s->cs_size != ULONG_MAX; s++) {
-               struct array_cache **alc;
                struct kmem_list3 *l3;
-               int r;
 
                l3 = s->cs_cachep->nodelists[q];
                if (!l3 || OFF_SLAB(s->cs_cachep))
                        continue;
-               lockdep_set_class(&l3->list_lock, &on_slab_l3_key);
-               alc = l3->alien;
-               /*
-                * FIXME: This check for BAD_ALIEN_MAGIC
-                * should go away when common slab code is taught to
-                * work even without alien caches.
-                * Currently, non NUMA code returns BAD_ALIEN_MAGIC
-                * for alloc_alien_cache,
-                */
-               if (!alc || (unsigned long)alc == BAD_ALIEN_MAGIC)
-                       continue;
-               for_each_node(r) {
-                       if (alc[r])
-                               lockdep_set_class(&alc[r]->lock,
-                                       &on_slab_alc_key);
-               }
+
+               slab_set_lock_classes(s->cs_cachep, &on_slab_l3_key,
+                               &on_slab_alc_key, q);
        }
 }
 
@@ -669,6 +701,14 @@ static void init_node_lock_keys(int q)
 static inline void init_lock_keys(void)
 {
 }
+
+static void slab_set_debugobj_lock_classes_node(struct kmem_cache *cachep, int node)
+{
+}
+
+static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep)
+{
+}
 #endif
 
 /*
@@ -1262,6 +1302,8 @@ static int __cpuinit cpuup_prepare(long cpu)
                spin_unlock_irq(&l3->list_lock);
                kfree(shared);
                free_alien_cache(alien);
+               if (cachep->flags & SLAB_DEBUG_OBJECTS)
+                       slab_set_debugobj_lock_classes_node(cachep, node);
        }
        init_node_lock_keys(node);
 
@@ -1492,11 +1534,10 @@ void __init kmem_cache_init(void)
        cache_cache.nodelists[node] = &initkmem_list3[CACHE_CACHE + node];
 
        /*
-        * struct kmem_cache size depends on nr_node_ids, which
-        * can be less than MAX_NUMNODES.
+        * struct kmem_cache size depends on nr_node_ids & nr_cpu_ids
         */
-       cache_cache.buffer_size = offsetof(struct kmem_cache, nodelists) +
-                                nr_node_ids * sizeof(struct kmem_list3 *);
+       cache_cache.buffer_size = offsetof(struct kmem_cache, array[nr_cpu_ids]) +
+                                 nr_node_ids * sizeof(struct kmem_list3 *);
 #if DEBUG
        cache_cache.obj_size = cache_cache.buffer_size;
 #endif
@@ -2308,6 +2349,7 @@ kmem_cache_create (const char *name, size_t size, size_t align,
        if (!cachep)
                goto oops;
 
+       cachep->nodelists = (struct kmem_list3 **)&cachep->array[nr_cpu_ids];
 #if DEBUG
        cachep->obj_size = size;
 
@@ -2424,6 +2466,16 @@ kmem_cache_create (const char *name, size_t size, size_t align,
                goto oops;
        }
 
+       if (flags & SLAB_DEBUG_OBJECTS) {
+               /*
+                * Would deadlock through slab_destroy()->call_rcu()->
+                * debug_object_activate()->kmem_cache_alloc().
+                */
+               WARN_ON_ONCE(flags & SLAB_DESTROY_BY_RCU);
+
+               slab_set_debugobj_lock_classes(cachep);
+       }
+
        /* cache setup completed, link it into the list */
        list_add(&cachep->next, &cache_chain);
 oops:
@@ -3153,12 +3205,11 @@ static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep,
        objp += obj_offset(cachep);
        if (cachep->ctor && cachep->flags & SLAB_POISON)
                cachep->ctor(objp);
-#if ARCH_SLAB_MINALIGN
-       if ((u32)objp & (ARCH_SLAB_MINALIGN-1)) {
+       if (ARCH_SLAB_MINALIGN &&
+           ((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) {
                printk(KERN_ERR "0x%p: not aligned to ARCH_SLAB_MINALIGN=%d\n",
-                      objp, ARCH_SLAB_MINALIGN);
+                      objp, (int)ARCH_SLAB_MINALIGN);
        }
-#endif
        return objp;
 }
 #else
@@ -3402,7 +3453,7 @@ __cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
        cache_alloc_debugcheck_before(cachep, flags);
        local_irq_save(save_flags);
 
-       if (nodeid == -1)
+       if (nodeid == NUMA_NO_NODE)
                nodeid = slab_node;
 
        if (unlikely(!cachep->nodelists[nodeid])) {
@@ -3933,7 +3984,7 @@ fail:
 
 struct ccupdate_struct {
        struct kmem_cache *cachep;
-       struct array_cache *new[NR_CPUS];
+       struct array_cache *new[0];
 };
 
 static void do_ccupdate_local(void *info)
@@ -3955,7 +4006,8 @@ static int do_tune_cpucache(struct kmem_cache *cachep, int limit,
        struct ccupdate_struct *new;
        int i;
 
-       new = kzalloc(sizeof(*new), gfp);
+       new = kzalloc(sizeof(*new) + nr_cpu_ids * sizeof(struct array_cache *),
+                     gfp);
        if (!new)
                return -ENOMEM;