2 * Generic VM initialization for x86-64 NUMA setups.
3 * Copyright 2002,2003 Andi Kleen, SuSE Labs.
5 #include <linux/kernel.h>
7 #include <linux/string.h>
8 #include <linux/init.h>
9 #include <linux/bootmem.h>
10 #include <linux/memblock.h>
11 #include <linux/mmzone.h>
12 #include <linux/ctype.h>
13 #include <linux/module.h>
14 #include <linux/nodemask.h>
15 #include <linux/sched.h>
16 #include <linux/acpi.h>
19 #include <asm/proto.h>
22 #include <asm/amd_nb.h>
24 #include "numa_internal.h"
26 struct pglist_data *node_data[MAX_NUMNODES] __read_mostly;
27 EXPORT_SYMBOL(node_data);
29 static struct numa_meminfo numa_meminfo
30 #ifndef CONFIG_MEMORY_HOTPLUG
35 static int numa_distance_cnt;
36 static u8 *numa_distance;
38 static int __init numa_add_memblk_to(int nid, u64 start, u64 end,
39 struct numa_meminfo *mi)
41 /* ignore zero length blks */
45 /* whine about and ignore invalid blks */
46 if (start > end || nid < 0 || nid >= MAX_NUMNODES) {
47 pr_warning("NUMA: Warning: invalid memblk node %d (%Lx-%Lx)\n",
52 if (mi->nr_blks >= NR_NODE_MEMBLKS) {
53 pr_err("NUMA: too many memblk ranges\n");
57 mi->blk[mi->nr_blks].start = start;
58 mi->blk[mi->nr_blks].end = end;
59 mi->blk[mi->nr_blks].nid = nid;
65 * numa_remove_memblk_from - Remove one numa_memblk from a numa_meminfo
66 * @idx: Index of memblk to remove
67 * @mi: numa_meminfo to remove memblk from
69 * Remove @idx'th numa_memblk from @mi by shifting @mi->blk[] and
70 * decrementing @mi->nr_blks.
72 void __init numa_remove_memblk_from(int idx, struct numa_meminfo *mi)
75 memmove(&mi->blk[idx], &mi->blk[idx + 1],
76 (mi->nr_blks - idx) * sizeof(mi->blk[0]));
80 * numa_add_memblk - Add one numa_memblk to numa_meminfo
81 * @nid: NUMA node ID of the new memblk
82 * @start: Start address of the new memblk
83 * @end: End address of the new memblk
85 * Add a new memblk to the default numa_meminfo.
88 * 0 on success, -errno on failure.
90 int __init numa_add_memblk(int nid, u64 start, u64 end)
92 return numa_add_memblk_to(nid, start, end, &numa_meminfo);
95 /* Initialize bootmem allocator for a node */
97 setup_node_bootmem(int nid, unsigned long start, unsigned long end)
99 const u64 nd_low = (u64)MAX_DMA_PFN << PAGE_SHIFT;
100 const u64 nd_high = (u64)max_pfn_mapped << PAGE_SHIFT;
101 const size_t nd_size = roundup(sizeof(pg_data_t), PAGE_SIZE);
106 * Don't confuse VM with a node that doesn't have the
107 * minimum amount of memory:
109 if (end && (end - start) < NODE_MIN_SIZE)
112 start = roundup(start, ZONE_ALIGN);
114 printk(KERN_INFO "Initmem setup node %d %016lx-%016lx\n",
118 * Try to allocate node data on local node and then fall back to
119 * all nodes. Never allocate in DMA zone.
121 nd_pa = memblock_x86_find_in_range_node(nid, nd_low, nd_high,
122 nd_size, SMP_CACHE_BYTES);
123 if (nd_pa == MEMBLOCK_ERROR)
124 nd_pa = memblock_find_in_range(nd_low, nd_high,
125 nd_size, SMP_CACHE_BYTES);
126 if (nd_pa == MEMBLOCK_ERROR) {
127 pr_err("Cannot find %lu bytes in node %d\n", nd_size, nid);
130 memblock_x86_reserve_range(nd_pa, nd_pa + nd_size, "NODE_DATA");
132 /* report and initialize */
133 printk(KERN_INFO " NODE_DATA [%016lx - %016lx]\n",
134 nd_pa, nd_pa + nd_size - 1);
135 tnid = early_pfn_to_nid(nd_pa >> PAGE_SHIFT);
137 printk(KERN_INFO " NODE_DATA(%d) on node %d\n", nid, tnid);
139 node_data[nid] = __va(nd_pa);
140 memset(NODE_DATA(nid), 0, sizeof(pg_data_t));
141 NODE_DATA(nid)->node_id = nid;
142 NODE_DATA(nid)->node_start_pfn = start >> PAGE_SHIFT;
143 NODE_DATA(nid)->node_spanned_pages = (end - start) >> PAGE_SHIFT;
145 node_set_online(nid);
149 * numa_cleanup_meminfo - Cleanup a numa_meminfo
150 * @mi: numa_meminfo to clean up
152 * Sanitize @mi by merging and removing unncessary memblks. Also check for
153 * conflicts and clear unused memblks.
156 * 0 on success, -errno on failure.
158 int __init numa_cleanup_meminfo(struct numa_meminfo *mi)
161 const u64 high = (u64)max_pfn << PAGE_SHIFT;
164 for (i = 0; i < mi->nr_blks; i++) {
165 struct numa_memblk *bi = &mi->blk[i];
167 /* make sure all blocks are inside the limits */
168 bi->start = max(bi->start, low);
169 bi->end = min(bi->end, high);
171 /* and there's no empty block */
172 if (bi->start >= bi->end) {
173 numa_remove_memblk_from(i--, mi);
177 for (j = i + 1; j < mi->nr_blks; j++) {
178 struct numa_memblk *bj = &mi->blk[j];
179 unsigned long start, end;
182 * See whether there are overlapping blocks. Whine
183 * about but allow overlaps of the same nid. They
184 * will be merged below.
186 if (bi->end > bj->start && bi->start < bj->end) {
187 if (bi->nid != bj->nid) {
188 pr_err("NUMA: node %d (%Lx-%Lx) overlaps with node %d (%Lx-%Lx)\n",
189 bi->nid, bi->start, bi->end,
190 bj->nid, bj->start, bj->end);
193 pr_warning("NUMA: Warning: node %d (%Lx-%Lx) overlaps with itself (%Lx-%Lx)\n",
194 bi->nid, bi->start, bi->end,
199 * Join together blocks on the same node, holes
200 * between which don't overlap with memory on other
203 if (bi->nid != bj->nid)
205 start = max(min(bi->start, bj->start), low);
206 end = min(max(bi->end, bj->end), high);
207 for (k = 0; k < mi->nr_blks; k++) {
208 struct numa_memblk *bk = &mi->blk[k];
210 if (bi->nid == bk->nid)
212 if (start < bk->end && end > bk->start)
217 printk(KERN_INFO "NUMA: Node %d [%Lx,%Lx) + [%Lx,%Lx) -> [%lx,%lx)\n",
218 bi->nid, bi->start, bi->end, bj->start, bj->end,
222 numa_remove_memblk_from(j--, mi);
226 for (i = mi->nr_blks; i < ARRAY_SIZE(mi->blk); i++) {
227 mi->blk[i].start = mi->blk[i].end = 0;
228 mi->blk[i].nid = NUMA_NO_NODE;
235 * Set nodes, which have memory in @mi, in *@nodemask.
237 static void __init numa_nodemask_from_meminfo(nodemask_t *nodemask,
238 const struct numa_meminfo *mi)
242 for (i = 0; i < ARRAY_SIZE(mi->blk); i++)
243 if (mi->blk[i].start != mi->blk[i].end &&
244 mi->blk[i].nid != NUMA_NO_NODE)
245 node_set(mi->blk[i].nid, *nodemask);
249 * numa_reset_distance - Reset NUMA distance table
251 * The current table is freed. The next numa_set_distance() call will
254 void __init numa_reset_distance(void)
256 size_t size = numa_distance_cnt * numa_distance_cnt * sizeof(numa_distance[0]);
258 /* numa_distance could be 1LU marking allocation failure, test cnt */
259 if (numa_distance_cnt)
260 memblock_x86_free_range(__pa(numa_distance),
261 __pa(numa_distance) + size);
262 numa_distance_cnt = 0;
263 numa_distance = NULL; /* enable table creation */
266 static int __init numa_alloc_distance(void)
268 nodemask_t nodes_parsed;
273 /* size the new table and allocate it */
274 nodes_parsed = numa_nodes_parsed;
275 numa_nodemask_from_meminfo(&nodes_parsed, &numa_meminfo);
277 for_each_node_mask(i, nodes_parsed)
280 size = cnt * cnt * sizeof(numa_distance[0]);
282 phys = memblock_find_in_range(0, (u64)max_pfn_mapped << PAGE_SHIFT,
284 if (phys == MEMBLOCK_ERROR) {
285 pr_warning("NUMA: Warning: can't allocate distance table!\n");
286 /* don't retry until explicitly reset */
287 numa_distance = (void *)1LU;
290 memblock_x86_reserve_range(phys, phys + size, "NUMA DIST");
292 numa_distance = __va(phys);
293 numa_distance_cnt = cnt;
295 /* fill with the default distances */
296 for (i = 0; i < cnt; i++)
297 for (j = 0; j < cnt; j++)
298 numa_distance[i * cnt + j] = i == j ?
299 LOCAL_DISTANCE : REMOTE_DISTANCE;
300 printk(KERN_DEBUG "NUMA: Initialized distance table, cnt=%d\n", cnt);
306 * numa_set_distance - Set NUMA distance from one NUMA to another
307 * @from: the 'from' node to set distance
308 * @to: the 'to' node to set distance
309 * @distance: NUMA distance
311 * Set the distance from node @from to @to to @distance. If distance table
312 * doesn't exist, one which is large enough to accommodate all the currently
313 * known nodes will be created.
315 * If such table cannot be allocated, a warning is printed and further
316 * calls are ignored until the distance table is reset with
317 * numa_reset_distance().
319 * If @from or @to is higher than the highest known node at the time of
320 * table creation or @distance doesn't make sense, the call is ignored.
321 * This is to allow simplification of specific NUMA config implementations.
323 void __init numa_set_distance(int from, int to, int distance)
325 if (!numa_distance && numa_alloc_distance() < 0)
328 if (from >= numa_distance_cnt || to >= numa_distance_cnt) {
329 printk_once(KERN_DEBUG "NUMA: Debug: distance out of bound, from=%d to=%d distance=%d\n",
334 if ((u8)distance != distance ||
335 (from == to && distance != LOCAL_DISTANCE)) {
336 pr_warn_once("NUMA: Warning: invalid distance parameter, from=%d to=%d distance=%d\n",
341 numa_distance[from * numa_distance_cnt + to] = distance;
344 int __node_distance(int from, int to)
346 if (from >= numa_distance_cnt || to >= numa_distance_cnt)
347 return from == to ? LOCAL_DISTANCE : REMOTE_DISTANCE;
348 return numa_distance[from * numa_distance_cnt + to];
350 EXPORT_SYMBOL(__node_distance);
353 * Sanity check to catch more bad NUMA configurations (they are amazingly
354 * common). Make sure the nodes cover all memory.
356 static bool __init numa_meminfo_cover_memory(const struct numa_meminfo *mi)
358 unsigned long numaram, e820ram;
362 for (i = 0; i < mi->nr_blks; i++) {
363 unsigned long s = mi->blk[i].start >> PAGE_SHIFT;
364 unsigned long e = mi->blk[i].end >> PAGE_SHIFT;
366 numaram -= __absent_pages_in_range(mi->blk[i].nid, s, e);
367 if ((long)numaram < 0)
371 e820ram = max_pfn - (memblock_x86_hole_size(0,
372 max_pfn << PAGE_SHIFT) >> PAGE_SHIFT);
373 /* We seem to lose 3 pages somewhere. Allow 1M of slack. */
374 if ((long)(e820ram - numaram) >= (1 << (20 - PAGE_SHIFT))) {
375 printk(KERN_ERR "NUMA: nodes only cover %luMB of your %luMB e820 RAM. Not used.\n",
376 (numaram << PAGE_SHIFT) >> 20,
377 (e820ram << PAGE_SHIFT) >> 20);
383 static int __init numa_register_memblks(struct numa_meminfo *mi)
387 /* Account for nodes with cpus and no memory */
388 node_possible_map = numa_nodes_parsed;
389 numa_nodemask_from_meminfo(&node_possible_map, mi);
390 if (WARN_ON(nodes_empty(node_possible_map)))
393 for (i = 0; i < mi->nr_blks; i++)
394 memblock_x86_register_active_regions(mi->blk[i].nid,
395 mi->blk[i].start >> PAGE_SHIFT,
396 mi->blk[i].end >> PAGE_SHIFT);
398 /* for out of order entries */
400 if (!numa_meminfo_cover_memory(mi))
403 /* Finally register nodes. */
404 for_each_node_mask(nid, node_possible_map) {
405 u64 start = (u64)max_pfn << PAGE_SHIFT;
408 for (i = 0; i < mi->nr_blks; i++) {
409 if (nid != mi->blk[i].nid)
411 start = min(mi->blk[i].start, start);
412 end = max(mi->blk[i].end, end);
416 setup_node_bootmem(nid, start, end);
423 * dummy_numma_init - Fallback dummy NUMA init
425 * Used if there's no underlying NUMA architecture, NUMA initialization
426 * fails, or NUMA is disabled on the command line.
428 * Must online at least one node and add memory blocks that cover all
429 * allowed memory. This function must not fail.
431 static int __init dummy_numa_init(void)
433 printk(KERN_INFO "%s\n",
434 numa_off ? "NUMA turned off" : "No NUMA configuration found");
435 printk(KERN_INFO "Faking a node at %016lx-%016lx\n",
436 0LU, max_pfn << PAGE_SHIFT);
438 node_set(0, numa_nodes_parsed);
439 numa_add_memblk(0, 0, (u64)max_pfn << PAGE_SHIFT);
444 static int __init numa_init(int (*init_func)(void))
449 for (i = 0; i < MAX_LOCAL_APIC; i++)
450 set_apicid_to_node(i, NUMA_NO_NODE);
452 nodes_clear(numa_nodes_parsed);
453 nodes_clear(node_possible_map);
454 nodes_clear(node_online_map);
455 memset(&numa_meminfo, 0, sizeof(numa_meminfo));
456 remove_all_active_ranges();
457 numa_reset_distance();
462 ret = numa_cleanup_meminfo(&numa_meminfo);
466 numa_emulation(&numa_meminfo, numa_distance_cnt);
468 ret = numa_register_memblks(&numa_meminfo);
472 for (i = 0; i < nr_cpu_ids; i++) {
473 int nid = early_cpu_to_node(i);
475 if (nid == NUMA_NO_NODE)
477 if (!node_online(nid))
484 void __init initmem_init(void)
487 #ifdef CONFIG_ACPI_NUMA
488 if (!numa_init(x86_acpi_numa_init))
491 #ifdef CONFIG_AMD_NUMA
492 if (!numa_init(amd_numa_init))
497 numa_init(dummy_numa_init);
500 unsigned long __init numa_free_all_bootmem(void)
502 unsigned long pages = 0;
505 for_each_online_node(i)
506 pages += free_all_bootmem_node(NODE_DATA(i));
508 pages += free_all_memory_core_early(MAX_NUMNODES);
513 #ifdef CONFIG_MEMORY_HOTPLUG
514 int memory_add_physaddr_to_nid(u64 start)
516 struct numa_meminfo *mi = &numa_meminfo;
517 int nid = mi->blk[0].nid;
520 for (i = 0; i < mi->nr_blks; i++)
521 if (mi->blk[i].start <= start && mi->blk[i].end > start)
522 nid = mi->blk[i].nid;
525 EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);