NFS: Eliminate nfs_get_user_pages()

[pandora-kernel.git] / mm / slab.c

diff --git a/mm/slab.c b/mm/slab.c
index 54bd2eb..98ac20b 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -331,6 +331,8 @@ static __always_inline int index_of(const size_t size)
        return 0;
 }
 
+static int slab_early_init = 1;
+
 #define INDEX_AC index_of(sizeof(struct arraycache_init))
 #define INDEX_L3 index_of(sizeof(struct kmem_list3))
 
@@ -592,6 +594,7 @@ static inline struct kmem_cache *page_get_cache(struct page *page)
 {
        if (unlikely(PageCompound(page)))
                page = (struct page *)page_private(page);
+       BUG_ON(!PageSlab(page));
        return (struct kmem_cache *)page->lru.next;
 }
 
@@ -604,6 +607,7 @@ static inline struct slab *page_get_slab(struct page *page)
 {
        if (unlikely(PageCompound(page)))
                page = (struct page *)page_private(page);
+       BUG_ON(!PageSlab(page));
        return (struct slab *)page->lru.prev;
 }
 
@@ -1374,6 +1378,8 @@ void __init kmem_cache_init(void)
                                NULL, NULL);
        }
 
+       slab_early_init = 0;
+
        while (sizes->cs_size != ULONG_MAX) {
                /*
                 * For performance, all the general caches are L1 aligned.
@@ -2104,8 +2110,12 @@ kmem_cache_create (const char *name, size_t size, size_t align,
 #endif
 #endif
 
-       /* Determine if the slab management is 'on' or 'off' slab. */
-       if (size >= (PAGE_SIZE >> 3))
+       /*
+        * Determine if the slab management is 'on' or 'off' slab.
+        * (bootstrapping cannot cope with offslab caches so don't do
+        * it too early on.)
+        */
+       if ((size >= (PAGE_SIZE >> 3)) && !slab_early_init)
                /*
                 * Size is large, assume best to place the slab management obj
                 * off-slab (should allow better packing of objs).
@@ -2636,6 +2646,28 @@ static void kfree_debugcheck(const void *objp)
        }
 }
 
+static inline void verify_redzone_free(struct kmem_cache *cache, void *obj)
+{
+       unsigned long redzone1, redzone2;
+
+       redzone1 = *dbg_redzone1(cache, obj);
+       redzone2 = *dbg_redzone2(cache, obj);
+
+       /*
+        * Redzone is ok.
+        */
+       if (redzone1 == RED_ACTIVE && redzone2 == RED_ACTIVE)
+               return;
+
+       if (redzone1 == RED_INACTIVE && redzone2 == RED_INACTIVE)
+               slab_error(cache, "double free detected");
+       else
+               slab_error(cache, "memory outside object was overwritten");
+
+       printk(KERN_ERR "%p: redzone 1:0x%lx, redzone 2:0x%lx.\n",
+                       obj, redzone1, redzone2);
+}
+
 static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp,
                                   void *caller)
 {
@@ -2647,27 +2679,10 @@ static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp,
        kfree_debugcheck(objp);
        page = virt_to_page(objp);
 
-       if (page_get_cache(page) != cachep) {
-               printk(KERN_ERR "mismatch in kmem_cache_free: expected "
-                               "cache %p, got %p\n",
-                      page_get_cache(page), cachep);
-               printk(KERN_ERR "%p is %s.\n", cachep, cachep->name);
-               printk(KERN_ERR "%p is %s.\n", page_get_cache(page),
-                      page_get_cache(page)->name);
-               WARN_ON(1);
-       }
        slabp = page_get_slab(page);
 
        if (cachep->flags & SLAB_RED_ZONE) {
-               if (*dbg_redzone1(cachep, objp) != RED_ACTIVE ||
-                               *dbg_redzone2(cachep, objp) != RED_ACTIVE) {
-                       slab_error(cachep, "double free, or memory outside"
-                                               " object was overwritten");
-                       printk(KERN_ERR "%p: redzone 1:0x%lx, "
-                                       "redzone 2:0x%lx.\n",
-                              objp, *dbg_redzone1(cachep, objp),
-                              *dbg_redzone2(cachep, objp));
-               }
+               verify_redzone_free(cachep, objp);
                *dbg_redzone1(cachep, objp) = RED_INACTIVE;
                *dbg_redzone2(cachep, objp) = RED_INACTIVE;
        }
@@ -3262,26 +3277,10 @@ EXPORT_SYMBOL(kmalloc_node);
 #endif
 
 /**
- * kmalloc - allocate memory
+ * __do_kmalloc - allocate memory
  * @size: how many bytes of memory are required.
- * @flags: the type of memory to allocate.
+ * @flags: the type of memory to allocate (see kmalloc).
  * @caller: function caller for debug tracking of the caller
- *
- * kmalloc is the normal method of allocating memory
- * in the kernel.
- *
- * The @flags argument may be one of:
- *
- * %GFP_USER - Allocate memory on behalf of user.  May sleep.
- *
- * %GFP_KERNEL - Allocate normal kernel ram.  May sleep.
- *
- * %GFP_ATOMIC - Allocation will not sleep.  Use inside interrupt handlers.
- *
- * Additionally, the %GFP_DMA flag may be set to indicate the memory
- * must be suitable for DMA.  This can mean different things on different
- * platforms.  For example, on i386, it means that the memory must come
- * from the first 16MB.
  */
 static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
                                          void *caller)
@@ -3379,6 +3378,8 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp)
 {
        unsigned long flags;
 
+       BUG_ON(virt_to_cache(objp) != cachep);
+
        local_irq_save(flags);
        __cache_free(cachep, objp);
        local_irq_restore(flags);