Merge branch 'stable-3.2' into pandora-3.2

[pandora-kernel.git] / mm / slub.c

diff --git a/mm/slub.c b/mm/slub.c
index fc719f7..dd62331 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -269,6 +269,11 @@ static inline void *get_freepointer(struct kmem_cache *s, void *object)
        return *(void **)(object + s->offset);
 }
 
+static void prefetch_freepointer(const struct kmem_cache *s, void *object)
+{
+       prefetch(object + s->offset);
+}
+
 static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
 {
        void *p;
@@ -1608,7 +1613,7 @@ static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags,
 
        do {
                cpuset_mems_cookie = get_mems_allowed();
-               zonelist = node_zonelist(slab_node(current->mempolicy), flags);
+               zonelist = node_zonelist(slab_node(), flags);
                for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
                        struct kmem_cache_node *n;
 
@@ -1873,18 +1878,24 @@ redo:
 /* Unfreeze all the cpu partial slabs */
 static void unfreeze_partials(struct kmem_cache *s)
 {
-       struct kmem_cache_node *n = NULL;
+       struct kmem_cache_node *n = NULL, *n2 = NULL;
        struct kmem_cache_cpu *c = this_cpu_ptr(s->cpu_slab);
        struct page *page, *discard_page = NULL;
 
        while ((page = c->partial)) {
-               enum slab_modes { M_PARTIAL, M_FREE };
-               enum slab_modes l, m;
                struct page new;
                struct page old;
 
                c->partial = page->next;
-               l = M_FREE;
+
+               n2 = get_node(s, page_to_nid(page));
+               if (n != n2) {
+                       if (n)
+                               spin_unlock(&n->list_lock);
+
+                       n = n2;
+                       spin_lock(&n->list_lock);
+               }
 
                do {
 
@@ -1897,40 +1908,17 @@ static void unfreeze_partials(struct kmem_cache *s)
 
                        new.frozen = 0;
 
-                       if (!new.inuse && (!n || n->nr_partial > s->min_partial))
-                               m = M_FREE;
-                       else {
-                               struct kmem_cache_node *n2 = get_node(s,
-                                                       page_to_nid(page));
-
-                               m = M_PARTIAL;
-                               if (n != n2) {
-                                       if (n)
-                                               spin_unlock(&n->list_lock);
-
-                                       n = n2;
-                                       spin_lock(&n->list_lock);
-                               }
-                       }
-
-                       if (l != m) {
-                               if (l == M_PARTIAL)
-                                       remove_partial(n, page);
-                               else
-                                       add_partial(n, page,
-                                               DEACTIVATE_TO_TAIL);
-
-                               l = m;
-                       }
-
                } while (!cmpxchg_double_slab(s, page,
                                old.freelist, old.counters,
                                new.freelist, new.counters,
                                "unfreezing slab"));
 
-               if (m == M_FREE) {
+               if (unlikely(!new.inuse && n->nr_partial > s->min_partial)) {
                        page->next = discard_page;
                        discard_page = page;
+               } else {
+                       add_partial(n, page, DEACTIVATE_TO_TAIL);
+                       stat(s, FREE_ADD_PARTIAL);
                }
        }
 
@@ -2310,6 +2298,8 @@ redo:
                object = __slab_alloc(s, gfpflags, node, addr, c);
 
        else {
+               void *next_object = get_freepointer_safe(s, object);
+
                /*
                 * The cmpxchg will only match if there was no additional
                 * operation and if we are on the right processor.
@@ -2325,11 +2315,12 @@ redo:
                if (unlikely(!irqsafe_cpu_cmpxchg_double(
                                s->cpu_slab->freelist, s->cpu_slab->tid,
                                object, tid,
-                               get_freepointer_safe(s, object), next_tid(tid)))) {
+                               next_object, next_tid(tid)))) {
 
                        note_cmpxchg_failure("slab_alloc", s, tid);
                        goto redo;
                }
+               prefetch_freepointer(s, next_object);
                stat(s, ALLOC_FASTPATH);
        }