unsigned long saved_max_pfn;
#endif
+bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata;
+
/* return the number of _pages_ that will be allocated for the boot bitmap */
unsigned long __init bootmem_bootmap_pages(unsigned long pages)
{
/*
* Called once to set up the allocator itself.
*/
-static unsigned long __init init_bootmem_core(pg_data_t *pgdat,
+static unsigned long __init init_bootmem_core(bootmem_data_t *bdata,
unsigned long mapstart, unsigned long start, unsigned long end)
{
- bootmem_data_t *bdata = pgdat->bdata;
unsigned long mapsize;
+ mminit_validate_memmodel_limits(&start, &end);
bdata->node_bootmem_map = phys_to_virt(PFN_PHYS(mapstart));
bdata->node_boot_start = PFN_PHYS(start);
bdata->node_low_pfn = end;
*
* NOTE: This function is _not_ reentrant.
*/
-void * __init
-__alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size,
- unsigned long align, unsigned long goal, unsigned long limit)
+static void * __init
+alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size,
+ unsigned long align, unsigned long goal, unsigned long limit)
{
unsigned long areasize, preferred;
unsigned long i, start = 0, incr, eidx, end_pfn;
void *node_bootmem_map;
if (!size) {
- printk("__alloc_bootmem_core(): zero-sized request\n");
+ printk("alloc_bootmem_core(): zero-sized request\n");
BUG();
}
BUG_ON(align & (align-1));
return ret;
}
-static unsigned long __init free_all_bootmem_core(pg_data_t *pgdat)
+static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
{
struct page *page;
unsigned long pfn;
- bootmem_data_t *bdata = pgdat->bdata;
- unsigned long i, count, total = 0;
+ unsigned long i, count;
unsigned long idx;
unsigned long *map;
int gofast = 0;
pfn = PFN_DOWN(bdata->node_boot_start);
idx = bdata->node_low_pfn - pfn;
map = bdata->node_bootmem_map;
- /* Check physaddr is O(LOG2(BITS_PER_LONG)) page aligned */
- if (bdata->node_boot_start == 0 ||
- ffs(bdata->node_boot_start) - PAGE_SHIFT > ffs(BITS_PER_LONG))
+ /*
+ * Check if we are aligned to BITS_PER_LONG pages. If so, we might
+ * be able to free page orders of that size at once.
+ */
+ if (!(pfn & (BITS_PER_LONG-1)))
gofast = 1;
+
for (i = 0; i < idx; ) {
unsigned long v = ~map[i / BITS_PER_LONG];
}
pfn += BITS_PER_LONG;
}
- total += count;
/*
* Now free the allocator bitmap itself, it's not
* needed anymore:
*/
page = virt_to_page(bdata->node_bootmem_map);
- count = 0;
idx = (get_mapsize(bdata) + PAGE_SIZE-1) >> PAGE_SHIFT;
- for (i = 0; i < idx; i++, page++) {
+ for (i = 0; i < idx; i++, page++)
__free_pages_bootmem(page, 0);
- count++;
- }
- total += count;
+ count += i;
bdata->node_bootmem_map = NULL;
- return total;
+ return count;
}
unsigned long __init init_bootmem_node(pg_data_t *pgdat, unsigned long freepfn,
unsigned long startpfn, unsigned long endpfn)
{
- return init_bootmem_core(pgdat, freepfn, startpfn, endpfn);
+ return init_bootmem_core(pgdat->bdata, freepfn, startpfn, endpfn);
}
int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
{
register_page_bootmem_info_node(pgdat);
- return free_all_bootmem_core(pgdat);
+ return free_all_bootmem_core(pgdat->bdata);
}
unsigned long __init init_bootmem(unsigned long start, unsigned long pages)
{
max_low_pfn = pages;
min_low_pfn = start;
- return init_bootmem_core(NODE_DATA(0), start, 0, pages);
+ return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages);
}
#ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE
unsigned long __init free_all_bootmem(void)
{
- return free_all_bootmem_core(NODE_DATA(0));
+ return free_all_bootmem_core(NODE_DATA(0)->bdata);
}
void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
void *ptr;
list_for_each_entry(bdata, &bdata_list, list) {
- ptr = __alloc_bootmem_core(bdata, size, align, goal, 0);
+ ptr = alloc_bootmem_core(bdata, size, align, goal, 0);
if (ptr)
return ptr;
}
{
void *ptr;
- ptr = __alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
+ ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
if (ptr)
return ptr;
goal = PFN_PHYS(pfn);
limit = PFN_PHYS(section_nr_to_pfn(section_nr + 1)) - 1;
pgdat = NODE_DATA(early_pfn_to_nid(pfn));
- ptr = __alloc_bootmem_core(pgdat->bdata, size, SMP_CACHE_BYTES, goal,
- limit);
+ ptr = alloc_bootmem_core(pgdat->bdata, size, SMP_CACHE_BYTES, goal,
+ limit);
if (!ptr)
return NULL;
void *ptr;
list_for_each_entry(bdata, &bdata_list, list) {
- ptr = __alloc_bootmem_core(bdata, size, align, goal,
- ARCH_LOW_ADDRESS_LIMIT);
+ ptr = alloc_bootmem_core(bdata, size, align, goal,
+ ARCH_LOW_ADDRESS_LIMIT);
if (ptr)
return ptr;
}
void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
unsigned long align, unsigned long goal)
{
- return __alloc_bootmem_core(pgdat->bdata, size, align, goal,
- ARCH_LOW_ADDRESS_LIMIT);
+ return alloc_bootmem_core(pgdat->bdata, size, align, goal,
+ ARCH_LOW_ADDRESS_LIMIT);
}