slab: shrink sizeof(struct kmem_cache)

Reduce high order allocations for some setups.
(NR_CPUS=4096 -> we need 64KB per kmem_cache struct)

We now allocate exact needed size (using nr_cpu_ids and nr_node_ids)

This also makes code a bit smaller on x86_64, since some field offsets
are less than the 127 limit :

Before patch :
# size mm/slab.o
   text    data     bss     dec     hex filename
  22605  361665      32  384302   5dd2e mm/slab.o

After patch :
# size mm/slab.o
   text    data     bss     dec     hex filename
  22349	 353473	   8224	 384046	  5dc2e	mm/slab.o

CC: Andrew Morton <akpm@linux-foundation.org>
Reported-by: Konstantin Khlebnikov <khlebnikov@openvz.org>
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Acked-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Pekka Enberg <penberg@kernel.org>

diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h
index d7f6311..d00e0ba 100644 (file)
--- a/include/linux/slab_def.h
+++ b/include/linux/slab_def.h
@@ -24,21 +24,19 @@
  */
 
 struct kmem_cache {
  */
 
 struct kmem_cache {

-/* 1) per-cpu data, touched during every alloc/free */
-       struct array_cache *array[NR_CPUS];
-/* 2) Cache tunables. Protected by cache_chain_mutex */
+/* 1) Cache tunables. Protected by cache_chain_mutex */

        unsigned int batchcount;
        unsigned int limit;
        unsigned int shared;
 
        unsigned int buffer_size;
        u32 reciprocal_buffer_size;
        unsigned int batchcount;
        unsigned int limit;
        unsigned int shared;
 
        unsigned int buffer_size;
        u32 reciprocal_buffer_size;

-/* 3) touched by every alloc & free from the backend */
+/* 2) touched by every alloc & free from the backend */

 
        unsigned int flags;             /* constant flags */
        unsigned int num;               /* # of objs per slab */
 
 
        unsigned int flags;             /* constant flags */
        unsigned int num;               /* # of objs per slab */
 

-/* 4) cache_grow/shrink */
+/* 3) cache_grow/shrink */

        /* order of pgs per slab (2^n) */
        unsigned int gfporder;
 
        /* order of pgs per slab (2^n) */
        unsigned int gfporder;
 


@@ -54,11 +52,11 @@ struct kmem_cache {
        /* constructor func */
        void (*ctor)(void *obj);
 
        /* constructor func */
        void (*ctor)(void *obj);
 

-/* 5) cache creation/removal */
+/* 4) cache creation/removal */

        const char *name;
        struct list_head next;
 
        const char *name;
        struct list_head next;
 

-/* 6) statistics */
+/* 5) statistics */

 #ifdef CONFIG_DEBUG_SLAB
        unsigned long num_active;
        unsigned long num_allocations;
 #ifdef CONFIG_DEBUG_SLAB
        unsigned long num_active;
        unsigned long num_allocations;


@@ -85,16 +83,18 @@ struct kmem_cache {
        int obj_size;
 #endif /* CONFIG_DEBUG_SLAB */
 
        int obj_size;
 #endif /* CONFIG_DEBUG_SLAB */
 

+/* 6) per-cpu/per-node data, touched during every alloc/free */

        /*
        /*

-        * We put nodelists[] at the end of kmem_cache, because we want to size
-        * this array to nr_node_ids slots instead of MAX_NUMNODES
+        * We put array[] at the end of kmem_cache, because we want to size
+        * this array to nr_cpu_ids slots instead of NR_CPUS

         * (see kmem_cache_init())
         * (see kmem_cache_init())

-        * We still use [MAX_NUMNODES] and not [1] or [0] because cache_cache
-        * is statically defined, so we reserve the max number of nodes.
+        * We still use [NR_CPUS] and not [1] or [0] because cache_cache
+        * is statically defined, so we reserve the max number of cpus.

         */
         */

-       struct kmem_list3 *nodelists[MAX_NUMNODES];
+       struct kmem_list3 **nodelists;
+       struct array_cache *array[NR_CPUS];

        /*
        /*

-        * Do not add fields after nodelists[]
+        * Do not add fields after array[]

         */
 };
 
         */
 };
 

diff --git a/mm/slab.c b/mm/slab.c
index ef8ceb7..c3cb359 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -574,7 +574,9 @@ static struct arraycache_init initarray_generic =
     { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
 
 /* internal cache of cache description objs */
     { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
 
 /* internal cache of cache description objs */

+static struct kmem_list3 *cache_cache_nodelists[MAX_NUMNODES];

 static struct kmem_cache cache_cache = {
 static struct kmem_cache cache_cache = {

+       .nodelists = cache_cache_nodelists,

        .batchcount = 1,
        .limit = BOOT_CPUCACHE_ENTRIES,
        .shared = 1,
        .batchcount = 1,
        .limit = BOOT_CPUCACHE_ENTRIES,
        .shared = 1,


@@ -1492,11 +1494,10 @@ void __init kmem_cache_init(void)
        cache_cache.nodelists[node] = &initkmem_list3[CACHE_CACHE + node];
 
        /*
        cache_cache.nodelists[node] = &initkmem_list3[CACHE_CACHE + node];
 
        /*

-        * struct kmem_cache size depends on nr_node_ids, which
-        * can be less than MAX_NUMNODES.
+        * struct kmem_cache size depends on nr_node_ids & nr_cpu_ids

         */
         */

-       cache_cache.buffer_size = offsetof(struct kmem_cache, nodelists) +
-                                nr_node_ids * sizeof(struct kmem_list3 *);
+       cache_cache.buffer_size = offsetof(struct kmem_cache, array[nr_cpu_ids]) +
+                                 nr_node_ids * sizeof(struct kmem_list3 *);

 #if DEBUG
        cache_cache.obj_size = cache_cache.buffer_size;
 #endif
 #if DEBUG
        cache_cache.obj_size = cache_cache.buffer_size;
 #endif


@@ -2308,6 +2309,7 @@ kmem_cache_create (const char *name, size_t size, size_t align,
        if (!cachep)
                goto oops;
 
        if (!cachep)
                goto oops;
 

+       cachep->nodelists = (struct kmem_list3 **)&cachep->array[nr_cpu_ids];

 #if DEBUG
        cachep->obj_size = size;
 
 #if DEBUG
        cachep->obj_size = size;