mm/cma_debug: fix debugging alloc/free interface

[pandora-kernel.git] / mm / page_alloc.c

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index f1f455a..ef19f22 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -18,6 +18,7 @@
 #include <linux/mm.h>
 #include <linux/swap.h>
 #include <linux/interrupt.h>
+#include <linux/rwsem.h>
 #include <linux/pagemap.h>
 #include <linux/jiffies.h>
 #include <linux/bootmem.h>
@@ -61,6 +62,7 @@
 #include <linux/hugetlb.h>
 #include <linux/sched/rt.h>
 #include <linux/page_owner.h>
+#include <linux/kthread.h>
 
 #include <asm/sections.h>
 #include <asm/tlbflush.h>
@@ -242,11 +244,9 @@ static inline void reset_deferred_meminit(pg_data_t *pgdat)
 }
 
 /* Returns true if the struct page for the pfn is uninitialised */
-static inline bool __defermem_init early_page_uninitialised(unsigned long pfn)
+static inline bool __meminit early_page_uninitialised(unsigned long pfn)
 {
-       int nid = early_pfn_to_nid(pfn);
-
-       if (pfn >= NODE_DATA(nid)->first_deferred_pfn)
+       if (pfn >= NODE_DATA(early_pfn_to_nid(pfn))->first_deferred_pfn)
                return true;
 
        return false;
@@ -839,7 +839,6 @@ static void __meminit __init_single_page(struct page *page, unsigned long pfn,
                                unsigned long zone, int nid)
 {
        set_page_links(page, zone, nid, pfn);
-       mminit_verify_page_links(page, zone, nid, pfn);
        init_page_count(page);
        page_mapcount_reset(page);
        page_cpupid_reset_last(page);
@@ -959,7 +958,7 @@ static void __free_pages_ok(struct page *page, unsigned int order)
        local_irq_restore(flags);
 }
 
-static void __defer_init __free_pages_boot_core(struct page *page,
+static void __init __free_pages_boot_core(struct page *page,
                                        unsigned long pfn, unsigned int order)
 {
        unsigned int nr_pages = 1 << order;
@@ -1032,7 +1031,7 @@ static inline bool __meminit meminit_pfn_in_nid(unsigned long pfn, int node,
 #endif
 
 
-void __defer_init __free_pages_bootmem(struct page *page, unsigned long pfn,
+void __init __free_pages_bootmem(struct page *page, unsigned long pfn,
                                                        unsigned int order)
 {
        if (early_page_uninitialised(pfn))
@@ -1041,7 +1040,7 @@ void __defer_init __free_pages_bootmem(struct page *page, unsigned long pfn,
 }
 
 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
-static void __defermem_init deferred_free_range(struct page *page,
+static void __init deferred_free_range(struct page *page,
                                        unsigned long pfn, int nr_pages)
 {
        int i;
@@ -1061,20 +1060,30 @@ static void __defermem_init deferred_free_range(struct page *page,
                __free_pages_boot_core(page, pfn, 0);
 }
 
+static __initdata DECLARE_RWSEM(pgdat_init_rwsem);
+
 /* Initialise remaining memory on a node */
-void __defermem_init deferred_init_memmap(int nid)
+static int __init deferred_init_memmap(void *data)
 {
+       pg_data_t *pgdat = data;
+       int nid = pgdat->node_id;
        struct mminit_pfnnid_cache nid_init_state = { };
        unsigned long start = jiffies;
        unsigned long nr_pages = 0;
        unsigned long walk_start, walk_end;
        int i, zid;
        struct zone *zone;
-       pg_data_t *pgdat = NODE_DATA(nid);
        unsigned long first_init_pfn = pgdat->first_deferred_pfn;
+       const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
 
-       if (first_init_pfn == ULONG_MAX)
-               return;
+       if (first_init_pfn == ULONG_MAX) {
+               up_read(&pgdat_init_rwsem);
+               return 0;
+       }
+
+       /* Bind memory initialisation thread to a local node if possible */
+       if (!cpumask_empty(cpumask))
+               set_cpus_allowed_ptr(current, cpumask);
 
        /* Sanity check boundaries */
        BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn);
@@ -1166,8 +1175,24 @@ free_range:
        /* Sanity check that the next zone really is unpopulated */
        WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone));
 
-       pr_info("kswapd %d initialised %lu pages in %ums\n", nid, nr_pages,
+       pr_info("node %d initialised, %lu pages in %ums\n", nid, nr_pages,
                                        jiffies_to_msecs(jiffies - start));
+       up_read(&pgdat_init_rwsem);
+       return 0;
+}
+
+void __init page_alloc_init_late(void)
+{
+       int nid;
+
+       for_each_node_state(nid, N_MEMORY) {
+               down_read(&pgdat_init_rwsem);
+               kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid);
+       }
+
+       /* Block until all are initialised */
+       down_write(&pgdat_init_rwsem);
+       up_write(&pgdat_init_rwsem);
 }
 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
 
@@ -1923,6 +1948,7 @@ void free_hot_cold_page_list(struct list_head *list, bool cold)
 void split_page(struct page *page, unsigned int order)
 {
        int i;
+       gfp_t gfp_mask;
 
        VM_BUG_ON_PAGE(PageCompound(page), page);
        VM_BUG_ON_PAGE(!page_count(page), page);
@@ -1936,10 +1962,11 @@ void split_page(struct page *page, unsigned int order)
                split_page(virt_to_page(page[0].shadow), order);
 #endif
 
-       set_page_owner(page, 0, 0);
+       gfp_mask = get_page_owner_gfp(page);
+       set_page_owner(page, 0, gfp_mask);
        for (i = 1; i < (1 << order); i++) {
                set_page_refcounted(page + i);
-               set_page_owner(page + i, 0, 0);
+               set_page_owner(page + i, 0, gfp_mask);
        }
 }
 EXPORT_SYMBOL_GPL(split_page);
@@ -1969,6 +1996,8 @@ int __isolate_free_page(struct page *page, unsigned int order)
        zone->free_area[order].nr_free--;
        rmv_page_order(page);
 
+       set_page_owner(page, order, __GFP_MOVABLE);
+
        /* Set the pageblock if the isolated page is at least a pageblock */
        if (order >= pageblock_order - 1) {
                struct page *endpage = page + (1 << order) - 1;
@@ -1980,7 +2009,7 @@ int __isolate_free_page(struct page *page, unsigned int order)
                }
        }
 
-       set_page_owner(page, order, 0);
+
        return 1UL << order;
 }