#include <linux/cache.h>
#include <linux/jiffies.h>
#include <linux/profile.h>
-#include <linux/lmb.h>
+#include <linux/bootmem.h>
+#include <linux/vmalloc.h>
#include <linux/cpu.h>
#include <asm/head.h>
#include <asm/ldc.h>
#include <asm/hypervisor.h>
+#include "cpumap.h"
+
int sparc64_multi_core __read_mostly;
DEFINE_PER_CPU(cpumask_t, cpu_sibling_map) = CPU_MASK_NONE;
cpu_clear(cpu, cpu_online_map);
ipi_call_unlock();
+ cpu_map_rebuild();
+
return 0;
}
{
}
-void __init real_setup_per_cpu_areas(void)
+/**
+ * pcpu_alloc_bootmem - NUMA friendly alloc_bootmem wrapper for percpu
+ * @cpu: cpu to allocate for
+ * @size: size allocation in bytes
+ * @align: alignment
+ *
+ * Allocate @size bytes aligned at @align for cpu @cpu. This wrapper
+ * does the right thing for NUMA regardless of the current
+ * configuration.
+ *
+ * RETURNS:
+ * Pointer to the allocated area on success, NULL on failure.
+ */
+static void * __init pcpu_alloc_bootmem(unsigned int cpu, unsigned long size,
+ unsigned long align)
{
- unsigned long base, shift, paddr, goal, size, i;
- char *ptr;
+ const unsigned long goal = __pa(MAX_DMA_ADDRESS);
+#ifdef CONFIG_NEED_MULTIPLE_NODES
+ int node = cpu_to_node(cpu);
+ void *ptr;
+
+ if (!node_online(node) || !NODE_DATA(node)) {
+ ptr = __alloc_bootmem(size, align, goal);
+ pr_info("cpu %d has no node %d or node-local memory\n",
+ cpu, node);
+ pr_debug("per cpu data for cpu%d %lu bytes at %016lx\n",
+ cpu, size, __pa(ptr));
+ } else {
+ ptr = __alloc_bootmem_node(NODE_DATA(node),
+ size, align, goal);
+ pr_debug("per cpu data for cpu%d %lu bytes on node%d at "
+ "%016lx\n", cpu, size, node, __pa(ptr));
+ }
+ return ptr;
+#else
+ return __alloc_bootmem(size, align, goal);
+#endif
+}
+
+static size_t pcpur_size __initdata;
+static void **pcpur_ptrs __initdata;
- /* Copy section for each CPU (we discard the original) */
- goal = PERCPU_ENOUGH_ROOM;
+static struct page * __init pcpur_get_page(unsigned int cpu, int pageno)
+{
+ size_t off = (size_t)pageno << PAGE_SHIFT;
- shift = PAGE_SHIFT;
- for (size = PAGE_SIZE; size < goal; size <<= 1UL)
- shift++;
+ if (off >= pcpur_size)
+ return NULL;
- paddr = lmb_alloc(size * NR_CPUS, PAGE_SIZE);
- if (!paddr) {
- prom_printf("Cannot allocate per-cpu memory.\n");
- prom_halt();
+ return virt_to_page(pcpur_ptrs[cpu] + off);
+}
+
+#define PCPU_CHUNK_SIZE (4UL * 1024UL * 1024UL)
+
+static void __init pcpu_map_range(unsigned long start, unsigned long end,
+ struct page *page)
+{
+ unsigned long pfn = page_to_pfn(page);
+ unsigned long pte_base;
+
+ BUG_ON((pfn<<PAGE_SHIFT)&(PCPU_CHUNK_SIZE - 1UL));
+
+ pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4U |
+ _PAGE_CP_4U | _PAGE_CV_4U |
+ _PAGE_P_4U | _PAGE_W_4U);
+ if (tlb_type == hypervisor)
+ pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4V |
+ _PAGE_CP_4V | _PAGE_CV_4V |
+ _PAGE_P_4V | _PAGE_W_4V);
+
+ while (start < end) {
+ pgd_t *pgd = pgd_offset_k(start);
+ unsigned long this_end;
+ pud_t *pud;
+ pmd_t *pmd;
+ pte_t *pte;
+
+ pud = pud_offset(pgd, start);
+ if (pud_none(*pud)) {
+ pmd_t *new;
+
+ new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
+ pud_populate(&init_mm, pud, new);
+ }
+
+ pmd = pmd_offset(pud, start);
+ if (!pmd_present(*pmd)) {
+ pte_t *new;
+
+ new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
+ pmd_populate_kernel(&init_mm, pmd, new);
+ }
+
+ pte = pte_offset_kernel(pmd, start);
+ this_end = (start + PMD_SIZE) & PMD_MASK;
+ if (this_end > end)
+ this_end = end;
+
+ while (start < this_end) {
+ unsigned long paddr = pfn << PAGE_SHIFT;
+
+ pte_val(*pte) = (paddr | pte_base);
+
+ start += PAGE_SIZE;
+ pte++;
+ pfn++;
+ }
+ }
+}
+
+void __init setup_per_cpu_areas(void)
+{
+ size_t dyn_size, static_size = __per_cpu_end - __per_cpu_start;
+ static struct vm_struct vm;
+ unsigned long delta, cpu;
+ size_t pcpu_unit_size;
+ size_t ptrs_size;
+
+ pcpur_size = PFN_ALIGN(static_size + PERCPU_MODULE_RESERVE +
+ PERCPU_DYNAMIC_RESERVE);
+ dyn_size = pcpur_size - static_size - PERCPU_MODULE_RESERVE;
+
+
+ ptrs_size = PFN_ALIGN(nr_cpu_ids * sizeof(pcpur_ptrs[0]));
+ pcpur_ptrs = alloc_bootmem(ptrs_size);
+
+ for_each_possible_cpu(cpu) {
+ pcpur_ptrs[cpu] = pcpu_alloc_bootmem(cpu, PCPU_CHUNK_SIZE,
+ PCPU_CHUNK_SIZE);
+
+ free_bootmem(__pa(pcpur_ptrs[cpu] + pcpur_size),
+ PCPU_CHUNK_SIZE - pcpur_size);
+
+ memcpy(pcpur_ptrs[cpu], __per_cpu_load, static_size);
}
- ptr = __va(paddr);
- base = ptr - __per_cpu_start;
+ /* allocate address and map */
+ vm.flags = VM_ALLOC;
+ vm.size = nr_cpu_ids * PCPU_CHUNK_SIZE;
+ vm_area_register_early(&vm, PCPU_CHUNK_SIZE);
+
+ for_each_possible_cpu(cpu) {
+ unsigned long start = (unsigned long) vm.addr;
+ unsigned long end;
- for (i = 0; i < NR_CPUS; i++, ptr += size) {
- __per_cpu_offset(i) = base + (i * size);
- memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start);
+ start += cpu * PCPU_CHUNK_SIZE;
+ end = start + PCPU_CHUNK_SIZE;
+ pcpu_map_range(start, end, virt_to_page(pcpur_ptrs[cpu]));
+ }
+
+ pcpu_unit_size = pcpu_setup_first_chunk(pcpur_get_page, static_size,
+ PERCPU_MODULE_RESERVE, dyn_size,
+ PCPU_CHUNK_SIZE, vm.addr, NULL);
+
+ free_bootmem(__pa(pcpur_ptrs), ptrs_size);
+
+ delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
+ for_each_possible_cpu(cpu) {
+ __per_cpu_offset(cpu) = delta + cpu * pcpu_unit_size;
}
/* Setup %g5 for the boot cpu. */
__local_per_cpu_offset = __per_cpu_offset(smp_processor_id());
+
+ of_fill_in_cpu_data();
+ if (tlb_type == hypervisor)
+ mdesc_fill_in_cpu_data(cpu_all_mask);
}
git.openpandora.org / pandora-kernel.git / blobdiff