mm: make gather_stats() type-safe and remove forward declaration

[pandora-kernel.git] / mm / slub.c

diff --git a/mm/slub.c b/mm/slub.c
index 8657ab8..4aad32d 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -64,7 +64,7 @@
  *   we must stay away from it for a while since we may cause a bouncing
  *   cacheline if we try to acquire the lock. So go onto the next slab.
  *   If all pages are busy then we may allocate a new slab instead of reusing
- *   a partial slab. A new slab has noone operating on it and thus there is
+ *   a partial slab. A new slab has no one operating on it and thus there is
  *   no danger of cacheline contention.
  *
  *   Interrupts are disabled during allocation and deallocation in order to
@@ -261,6 +261,18 @@ static inline void *get_freepointer(struct kmem_cache *s, void *object)
        return *(void **)(object + s->offset);
 }
 
+static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
+{
+       void *p;
+
+#ifdef CONFIG_DEBUG_PAGEALLOC
+       probe_kernel_read(&p, (void **)(object + s->offset), sizeof(p));
+#else
+       p = get_freepointer(s, object);
+#endif
+       return p;
+}
+
 static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
 {
        *(void **)(object + s->offset) = fp;
@@ -1833,7 +1845,6 @@ new_slab:
        page = get_partial(s, gfpflags, node);
        if (page) {
                stat(s, ALLOC_FROM_PARTIAL);
-load_from_page:
                c->node = page_to_nid(page);
                c->page = page;
                goto load_freelist;
@@ -1856,8 +1867,9 @@ load_from_page:
 
                slab_lock(page);
                __SetPageSlubFrozen(page);
-
-               goto load_from_page;
+               c->node = page_to_nid(page);
+               c->page = page;
+               goto load_freelist;
        }
        if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit())
                slab_out_of_memory(s, gfpflags, node);
@@ -1869,8 +1881,11 @@ debug:
 
        page->inuse++;
        page->freelist = get_freepointer(s, object);
+       deactivate_slab(s, c);
+       c->page = NULL;
        c->node = NUMA_NO_NODE;
-       goto unlock_out;
+       local_irq_restore(flags);
+       return object;
 }
 
 /*
@@ -1919,7 +1934,7 @@ redo:
 
        else {
                /*
-                * The cmpxchg will only match if there was no additonal
+                * The cmpxchg will only match if there was no additional
                 * operation and if we are on the right processor.
                 *
                 * The cmpxchg does the following atomically (without lock semantics!)
@@ -1930,10 +1945,10 @@ redo:
                 * Since this is without lock semantics the protection is only against
                 * code executing on this cpu *not* from access by other cpus.
                 */
-               if (unlikely(!this_cpu_cmpxchg_double(
+               if (unlikely(!irqsafe_cpu_cmpxchg_double(
                                s->cpu_slab->freelist, s->cpu_slab->tid,
                                object, tid,
-                               get_freepointer(s, object), next_tid(tid)))) {
+                               get_freepointer_safe(s, object), next_tid(tid)))) {
 
                        note_cmpxchg_failure("slab_alloc", s, tid);
                        goto redo;
@@ -2100,10 +2115,10 @@ redo:
        tid = c->tid;
        barrier();
 
-       if (likely(page == c->page && c->node != NUMA_NO_NODE)) {
+       if (likely(page == c->page)) {
                set_freepointer(s, object, c->freelist);
 
-               if (unlikely(!this_cpu_cmpxchg_double(
+               if (unlikely(!irqsafe_cpu_cmpxchg_double(
                                s->cpu_slab->freelist, s->cpu_slab->tid,
                                c->freelist, tid,
                                object, next_tid(tid)))) {
@@ -3498,7 +3513,7 @@ void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
 
        ret = slab_alloc(s, gfpflags, NUMA_NO_NODE, caller);
 
-       /* Honor the call site pointer we recieved. */
+       /* Honor the call site pointer we received. */
        trace_kmalloc(caller, ret, size, s->size, gfpflags);
 
        return ret;
@@ -3528,7 +3543,7 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
 
        ret = slab_alloc(s, gfpflags, node, caller);
 
-       /* Honor the call site pointer we recieved. */
+       /* Honor the call site pointer we received. */
        trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node);
 
        return ret;