2 * x86 SMP booting functions
4 * (c) 1995 Alan Cox, Building #3 <alan@redhat.com>
5 * (c) 1998, 1999, 2000 Ingo Molnar <mingo@redhat.com>
6 * Copyright 2001 Andi Kleen, SuSE Labs.
8 * Much of the core SMP work is based on previous work by Thomas Radke, to
9 * whom a great many thanks are extended.
11 * Thanks to Intel for making available several different Pentium,
12 * Pentium Pro and Pentium-II/Xeon MP machines.
13 * Original development of Linux SMP code supported by Caldera.
15 * This code is released under the GNU General Public License version 2
18 * Felix Koop : NR_CPUS used properly
19 * Jose Renau : Handle single CPU case.
20 * Alan Cox : By repeated request 8) - Total BogoMIP report.
21 * Greg Wright : Fix for kernel stacks panic.
22 * Erich Boleyn : MP v1.4 and additional changes.
23 * Matthias Sattler : Changes for 2.1 kernel map.
24 * Michel Lespinasse : Changes for 2.1 kernel map.
25 * Michael Chastain : Change trampoline.S to gnu as.
26 * Alan Cox : Dumb bug: 'B' step PPro's are fine
27 * Ingo Molnar : Added APIC timers, based on code
29 * Ingo Molnar : various cleanups and rewrites
30 * Tigran Aivazian : fixed "0.00 in /proc/uptime on SMP" bug.
31 * Maciej W. Rozycki : Bits for genuine 82489DX APICs
32 * Andi Kleen : Changed for SMP boot into long mode.
33 * Rusty Russell : Hacked into shape for new "hotplug" boot process.
34 * Andi Kleen : Converted to new state machine.
36 * Probably mostly hotplug CPU ready now.
37 * Ashok Raj : CPU hotplug support
41 #include <linux/config.h>
42 #include <linux/init.h>
45 #include <linux/kernel_stat.h>
46 #include <linux/smp_lock.h>
47 #include <linux/irq.h>
48 #include <linux/bootmem.h>
49 #include <linux/thread_info.h>
50 #include <linux/module.h>
52 #include <linux/delay.h>
53 #include <linux/mc146818rtc.h>
55 #include <asm/pgalloc.h>
57 #include <asm/kdebug.h>
58 #include <asm/tlbflush.h>
59 #include <asm/proto.h>
62 /* Number of siblings per CPU package */
63 int smp_num_siblings = 1;
64 /* Package ID of each logical CPU */
65 u8 phys_proc_id[NR_CPUS] __read_mostly = { [0 ... NR_CPUS-1] = BAD_APICID };
66 u8 cpu_core_id[NR_CPUS] __read_mostly = { [0 ... NR_CPUS-1] = BAD_APICID };
67 EXPORT_SYMBOL(phys_proc_id);
68 EXPORT_SYMBOL(cpu_core_id);
70 /* Bitmask of currently online CPUs */
71 cpumask_t cpu_online_map __read_mostly;
73 EXPORT_SYMBOL(cpu_online_map);
76 * Private maps to synchronize booting between AP and BP.
77 * Probably not needed anymore, but it makes for easier debugging. -AK
79 cpumask_t cpu_callin_map;
80 cpumask_t cpu_callout_map;
82 cpumask_t cpu_possible_map;
83 EXPORT_SYMBOL(cpu_possible_map);
85 /* Per CPU bogomips and other parameters */
86 struct cpuinfo_x86 cpu_data[NR_CPUS] __cacheline_aligned;
88 /* Set when the idlers are all forked */
89 int smp_threads_ready;
91 cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly;
92 cpumask_t cpu_core_map[NR_CPUS] __read_mostly;
93 EXPORT_SYMBOL(cpu_core_map);
96 * Trampoline 80x86 program as an array.
99 extern unsigned char trampoline_data[];
100 extern unsigned char trampoline_end[];
102 /* State of each CPU */
103 DEFINE_PER_CPU(int, cpu_state) = { 0 };
106 * Store all idle threads, this can be reused instead of creating
107 * a new thread. Also avoids complicated thread destroy functionality
110 struct task_struct *idle_thread_array[NR_CPUS] __cpuinitdata ;
112 #define get_idle_for_cpu(x) (idle_thread_array[(x)])
113 #define set_idle_for_cpu(x,p) (idle_thread_array[(x)] = (p))
116 * Currently trivial. Write the real->protected mode
117 * bootstrap into the page concerned. The caller
118 * has made sure it's suitably aligned.
121 static unsigned long __cpuinit setup_trampoline(void)
123 void *tramp = __va(SMP_TRAMPOLINE_BASE);
124 memcpy(tramp, trampoline_data, trampoline_end - trampoline_data);
125 return virt_to_phys(tramp);
129 * The bootstrap kernel entry code has set these up. Save them for
133 static void __cpuinit smp_store_cpu_info(int id)
135 struct cpuinfo_x86 *c = cpu_data + id;
143 * New Funky TSC sync algorithm borrowed from IA64.
144 * Main advantage is that it doesn't reset the TSCs fully and
145 * in general looks more robust and it works better than my earlier
146 * attempts. I believe it was written by David Mosberger. Some minor
147 * adjustments for x86-64 by me -AK
149 * Original comment reproduced below.
151 * Synchronize TSC of the current (slave) CPU with the TSC of the
152 * MASTER CPU (normally the time-keeper CPU). We use a closed loop to
153 * eliminate the possibility of unaccounted-for errors (such as
154 * getting a machine check in the middle of a calibration step). The
155 * basic idea is for the slave to ask the master what itc value it has
156 * and to read its own itc before and after the master responds. Each
157 * iteration gives us three timestamps:
170 * The goal is to adjust the slave's TSC such that tm falls exactly
171 * half-way between t0 and t1. If we achieve this, the clocks are
172 * synchronized provided the interconnect between the slave and the
173 * master is symmetric. Even if the interconnect were asymmetric, we
174 * would still know that the synchronization error is smaller than the
175 * roundtrip latency (t0 - t1).
177 * When the interconnect is quiet and symmetric, this lets us
178 * synchronize the TSC to within one or two cycles. However, we can
179 * only *guarantee* that the synchronization is accurate to within a
180 * round-trip time, which is typically in the range of several hundred
181 * cycles (e.g., ~500 cycles). In practice, this means that the TSCs
182 * are usually almost perfectly synchronized, but we shouldn't assume
183 * that the accuracy is much better than half a micro second or so.
185 * [there are other errors like the latency of RDTSC and of the
186 * WRMSR. These can also account to hundreds of cycles. So it's
187 * probably worse. It claims 153 cycles error on a dual Opteron,
188 * but I suspect the numbers are actually somewhat worse -AK]
192 #define SLAVE (SMP_CACHE_BYTES/8)
194 /* Intentionally don't use cpu_relax() while TSC synchronization
195 because we don't want to go into funky power save modi or cause
196 hypervisors to schedule us away. Going to sleep would likely affect
197 latency and low latency is the primary objective here. -AK */
198 #define no_cpu_relax() barrier()
200 static __cpuinitdata DEFINE_SPINLOCK(tsc_sync_lock);
201 static volatile __cpuinitdata unsigned long go[SLAVE + 1];
202 static int notscsync __cpuinitdata;
204 #undef DEBUG_TSC_SYNC
206 #define NUM_ROUNDS 64 /* magic value */
207 #define NUM_ITERS 5 /* likewise */
209 /* Callback on boot CPU */
210 static __cpuinit void sync_master(void *arg)
212 unsigned long flags, i;
216 local_irq_save(flags);
218 for (i = 0; i < NUM_ROUNDS*NUM_ITERS; ++i) {
225 local_irq_restore(flags);
229 * Return the number of cycles by which our tsc differs from the tsc
230 * on the master (time-keeper) CPU. A positive number indicates our
231 * tsc is ahead of the master, negative that it is behind.
234 get_delta(long *rt, long *master)
236 unsigned long best_t0 = 0, best_t1 = ~0UL, best_tm = 0;
237 unsigned long tcenter, t0, t1, tm;
240 for (i = 0; i < NUM_ITERS; ++i) {
243 while (!(tm = go[SLAVE]))
248 if (t1 - t0 < best_t1 - best_t0)
249 best_t0 = t0, best_t1 = t1, best_tm = tm;
252 *rt = best_t1 - best_t0;
253 *master = best_tm - best_t0;
255 /* average best_t0 and best_t1 without overflow: */
256 tcenter = (best_t0/2 + best_t1/2);
257 if (best_t0 % 2 + best_t1 % 2 == 2)
259 return tcenter - best_tm;
262 static __cpuinit void sync_tsc(unsigned int master)
265 long delta, adj, adjust_latency = 0;
266 unsigned long flags, rt, master_time_stamp, bound;
267 #ifdef DEBUG_TSC_SYNC
268 static struct syncdebug {
269 long rt; /* roundtrip time */
270 long master; /* master's timestamp */
271 long diff; /* difference between midpoint and master's timestamp */
272 long lat; /* estimate of tsc adjustment latency */
273 } t[NUM_ROUNDS] __cpuinitdata;
276 printk(KERN_INFO "CPU %d: Syncing TSC to CPU %u.\n",
277 smp_processor_id(), master);
281 /* It is dangerous to broadcast IPI as cpus are coming up,
282 * as they may not be ready to accept them. So since
283 * we only need to send the ipi to the boot cpu direct
284 * the message, and avoid the race.
286 smp_call_function_single(master, sync_master, NULL, 1, 0);
288 while (go[MASTER]) /* wait for master to be ready */
291 spin_lock_irqsave(&tsc_sync_lock, flags);
293 for (i = 0; i < NUM_ROUNDS; ++i) {
294 delta = get_delta(&rt, &master_time_stamp);
296 done = 1; /* let's lock on to this... */
303 adjust_latency += -delta;
304 adj = -delta + adjust_latency/4;
309 wrmsrl(MSR_IA32_TSC, t + adj);
311 #ifdef DEBUG_TSC_SYNC
313 t[i].master = master_time_stamp;
315 t[i].lat = adjust_latency/4;
319 spin_unlock_irqrestore(&tsc_sync_lock, flags);
321 #ifdef DEBUG_TSC_SYNC
322 for (i = 0; i < NUM_ROUNDS; ++i)
323 printk("rt=%5ld master=%5ld diff=%5ld adjlat=%5ld\n",
324 t[i].rt, t[i].master, t[i].diff, t[i].lat);
328 "CPU %d: synchronized TSC with CPU %u (last diff %ld cycles, "
329 "maxerr %lu cycles)\n",
330 smp_processor_id(), master, delta, rt);
333 static void __cpuinit tsc_sync_wait(void)
335 if (notscsync || !cpu_has_tsc)
340 static __init int notscsync_setup(char *s)
345 __setup("notscsync", notscsync_setup);
347 static atomic_t init_deasserted __cpuinitdata;
350 * Report back to the Boot Processor.
353 void __cpuinit smp_callin(void)
356 unsigned long timeout;
359 * If waken up by an INIT in an 82489DX configuration
360 * we may get here before an INIT-deassert IPI reaches
361 * our local APIC. We have to wait for the IPI or we'll
362 * lock up on an APIC access.
364 while (!atomic_read(&init_deasserted))
368 * (This works even if the APIC is not enabled.)
370 phys_id = GET_APIC_ID(apic_read(APIC_ID));
371 cpuid = smp_processor_id();
372 if (cpu_isset(cpuid, cpu_callin_map)) {
373 panic("smp_callin: phys CPU#%d, CPU#%d already present??\n",
376 Dprintk("CPU#%d (phys ID: %d) waiting for CALLOUT\n", cpuid, phys_id);
379 * STARTUP IPIs are fragile beasts as they might sometimes
380 * trigger some glue motherboard logic. Complete APIC bus
381 * silence for 1 second, this overestimates the time the
382 * boot CPU is spending to send the up to 2 STARTUP IPIs
383 * by a factor of two. This should be enough.
387 * Waiting 2s total for startup (udelay is not yet working)
389 timeout = jiffies + 2*HZ;
390 while (time_before(jiffies, timeout)) {
392 * Has the boot CPU finished it's STARTUP sequence?
394 if (cpu_isset(cpuid, cpu_callout_map))
399 if (!time_before(jiffies, timeout)) {
400 panic("smp_callin: CPU%d started up but did not get a callout!\n",
405 * the boot CPU has finished the init stage and is spinning
406 * on callin_map until we finish. We are free to set up this
407 * CPU, first the APIC. (this is probably redundant on most
411 Dprintk("CALLIN, before setup_local_APIC().\n");
418 Dprintk("Stack at about %p\n",&cpuid);
420 disable_APIC_timer();
423 * Save our processor parameters
425 smp_store_cpu_info(cpuid);
428 * Allow the master to continue.
430 cpu_set(cpuid, cpu_callin_map);
433 static inline void set_cpu_sibling_map(int cpu)
437 if (smp_num_siblings > 1) {
439 if (cpu_core_id[cpu] == cpu_core_id[i]) {
440 cpu_set(i, cpu_sibling_map[cpu]);
441 cpu_set(cpu, cpu_sibling_map[i]);
445 cpu_set(cpu, cpu_sibling_map[cpu]);
448 if (current_cpu_data.x86_num_cores > 1) {
450 if (phys_proc_id[cpu] == phys_proc_id[i]) {
451 cpu_set(i, cpu_core_map[cpu]);
452 cpu_set(cpu, cpu_core_map[i]);
456 cpu_core_map[cpu] = cpu_sibling_map[cpu];
461 * Setup code on secondary processor (after comming out of the trampoline)
463 void __cpuinit start_secondary(void)
466 * Dont put anything before smp_callin(), SMP
467 * booting is too fragile that we want to limit the
468 * things done here to the most necessary things.
473 /* otherwise gcc will move up the smp_processor_id before the cpu_init */
476 Dprintk("cpu %d: setting up apic clock\n", smp_processor_id());
477 setup_secondary_APIC_clock();
479 Dprintk("cpu %d: enabling apic timer\n", smp_processor_id());
481 if (nmi_watchdog == NMI_IO_APIC) {
482 disable_8259A_irq(0);
483 enable_NMI_through_LVT0(NULL);
490 * The sibling maps must be set before turing the online map on for
493 set_cpu_sibling_map(smp_processor_id());
496 * Wait for TSC sync to not schedule things before.
497 * We still process interrupts, which could see an inconsistent
498 * time in that window unfortunately.
499 * Do this here because TSC sync has global unprotected state.
504 * We need to hold call_lock, so there is no inconsistency
505 * between the time smp_call_function() determines number of
506 * IPI receipients, and the time when the determination is made
507 * for which cpus receive the IPI in genapic_flat.c. Holding this
508 * lock helps us to not include this cpu in a currently in progress
509 * smp_call_function().
511 lock_ipi_call_lock();
514 * Allow the master to continue.
516 cpu_set(smp_processor_id(), cpu_online_map);
517 per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
518 unlock_ipi_call_lock();
523 extern volatile unsigned long init_rsp;
524 extern void (*initial_code)(void);
527 static void inquire_remote_apic(int apicid)
529 unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
530 char *names[] = { "ID", "VERSION", "SPIV" };
533 printk(KERN_INFO "Inquiring remote APIC #%d...\n", apicid);
535 for (i = 0; i < sizeof(regs) / sizeof(*regs); i++) {
536 printk("... APIC #%d %s: ", apicid, names[i]);
541 apic_wait_icr_idle();
543 apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(apicid));
544 apic_write(APIC_ICR, APIC_DM_REMRD | regs[i]);
549 status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
550 } while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);
553 case APIC_ICR_RR_VALID:
554 status = apic_read(APIC_RRR);
555 printk("%08x\n", status);
565 * Kick the secondary to wake up.
567 static int __cpuinit wakeup_secondary_via_INIT(int phys_apicid, unsigned int start_rip)
569 unsigned long send_status = 0, accept_status = 0;
570 int maxlvt, timeout, num_starts, j;
572 Dprintk("Asserting INIT.\n");
575 * Turn INIT on target chip
577 apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
582 apic_write(APIC_ICR, APIC_INT_LEVELTRIG | APIC_INT_ASSERT
585 Dprintk("Waiting for send to finish...\n");
590 send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
591 } while (send_status && (timeout++ < 1000));
595 Dprintk("Deasserting INIT.\n");
598 apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
601 apic_write(APIC_ICR, APIC_INT_LEVELTRIG | APIC_DM_INIT);
603 Dprintk("Waiting for send to finish...\n");
608 send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
609 } while (send_status && (timeout++ < 1000));
611 atomic_set(&init_deasserted, 1);
614 * Should we send STARTUP IPIs ?
616 * Determine this based on the APIC version.
617 * If we don't have an integrated APIC, don't send the STARTUP IPIs.
619 if (APIC_INTEGRATED(apic_version[phys_apicid]))
625 * Run STARTUP IPI loop.
627 Dprintk("#startup loops: %d.\n", num_starts);
629 maxlvt = get_maxlvt();
631 for (j = 1; j <= num_starts; j++) {
632 Dprintk("Sending STARTUP #%d.\n",j);
633 apic_read_around(APIC_SPIV);
634 apic_write(APIC_ESR, 0);
636 Dprintk("After apic_write.\n");
643 apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
645 /* Boot on the stack */
646 /* Kick the second */
647 apic_write(APIC_ICR, APIC_DM_STARTUP | (start_rip >> 12));
650 * Give the other CPU some time to accept the IPI.
654 Dprintk("Startup point 1.\n");
656 Dprintk("Waiting for send to finish...\n");
661 send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
662 } while (send_status && (timeout++ < 1000));
665 * Give the other CPU some time to accept the IPI.
669 * Due to the Pentium erratum 3AP.
672 apic_read_around(APIC_SPIV);
673 apic_write(APIC_ESR, 0);
675 accept_status = (apic_read(APIC_ESR) & 0xEF);
676 if (send_status || accept_status)
679 Dprintk("After Startup.\n");
682 printk(KERN_ERR "APIC never delivered???\n");
684 printk(KERN_ERR "APIC delivery error (%lx).\n", accept_status);
686 return (send_status | accept_status);
690 struct task_struct *idle;
691 struct completion done;
695 void do_fork_idle(void *_c_idle)
697 struct create_idle *c_idle = _c_idle;
699 c_idle->idle = fork_idle(c_idle->cpu);
700 complete(&c_idle->done);
706 static int __cpuinit do_boot_cpu(int cpu, int apicid)
708 unsigned long boot_error;
710 unsigned long start_rip;
711 struct create_idle c_idle = {
713 .done = COMPLETION_INITIALIZER(c_idle.done),
715 DECLARE_WORK(work, do_fork_idle, &c_idle);
717 c_idle.idle = get_idle_for_cpu(cpu);
720 c_idle.idle->thread.rsp = (unsigned long) (((struct pt_regs *)
721 (THREAD_SIZE + (unsigned long) c_idle.idle->thread_info)) - 1);
722 init_idle(c_idle.idle, cpu);
727 * During cold boot process, keventd thread is not spun up yet.
728 * When we do cpu hot-add, we create idle threads on the fly, we should
729 * not acquire any attributes from the calling context. Hence the clean
730 * way to create kernel_threads() is to do that from keventd().
731 * We do the current_is_keventd() due to the fact that ACPI notifier
732 * was also queuing to keventd() and when the caller is already running
733 * in context of keventd(), we would end up with locking up the keventd
736 if (!keventd_up() || current_is_keventd())
737 work.func(work.data);
739 schedule_work(&work);
740 wait_for_completion(&c_idle.done);
743 if (IS_ERR(c_idle.idle)) {
744 printk("failed fork for CPU %d\n", cpu);
745 return PTR_ERR(c_idle.idle);
748 set_idle_for_cpu(cpu, c_idle.idle);
752 cpu_pda[cpu].pcurrent = c_idle.idle;
754 start_rip = setup_trampoline();
756 init_rsp = c_idle.idle->thread.rsp;
757 per_cpu(init_tss,cpu).rsp0 = init_rsp;
758 initial_code = start_secondary;
759 clear_ti_thread_flag(c_idle.idle->thread_info, TIF_FORK);
761 printk(KERN_INFO "Booting processor %d/%d APIC 0x%x\n", cpu,
762 cpus_weight(cpu_present_map),
766 * This grunge runs the startup process for
767 * the targeted processor.
770 atomic_set(&init_deasserted, 0);
772 Dprintk("Setting warm reset code and vector.\n");
774 CMOS_WRITE(0xa, 0xf);
777 *((volatile unsigned short *) phys_to_virt(0x469)) = start_rip >> 4;
779 *((volatile unsigned short *) phys_to_virt(0x467)) = start_rip & 0xf;
783 * Be paranoid about clearing APIC errors.
785 if (APIC_INTEGRATED(apic_version[apicid])) {
786 apic_read_around(APIC_SPIV);
787 apic_write(APIC_ESR, 0);
792 * Status is now clean
797 * Starting actual IPI sequence...
799 boot_error = wakeup_secondary_via_INIT(apicid, start_rip);
803 * allow APs to start initializing.
805 Dprintk("Before Callout %d.\n", cpu);
806 cpu_set(cpu, cpu_callout_map);
807 Dprintk("After Callout %d.\n", cpu);
810 * Wait 5s total for a response
812 for (timeout = 0; timeout < 50000; timeout++) {
813 if (cpu_isset(cpu, cpu_callin_map))
814 break; /* It has booted */
818 if (cpu_isset(cpu, cpu_callin_map)) {
819 /* number CPUs logically, starting from 1 (BSP is 0) */
820 Dprintk("CPU has booted.\n");
823 if (*((volatile unsigned char *)phys_to_virt(SMP_TRAMPOLINE_BASE))
825 /* trampoline started but...? */
826 printk("Stuck ??\n");
828 /* trampoline code not run */
829 printk("Not responding.\n");
831 inquire_remote_apic(apicid);
836 cpu_clear(cpu, cpu_callout_map); /* was set here (do_boot_cpu()) */
837 clear_bit(cpu, &cpu_initialized); /* was set by cpu_init() */
838 cpu_clear(cpu, cpu_present_map);
839 cpu_clear(cpu, cpu_possible_map);
840 x86_cpu_to_apicid[cpu] = BAD_APICID;
841 x86_cpu_to_log_apicid[cpu] = BAD_APICID;
848 cycles_t cacheflush_time;
849 unsigned long cache_decay_ticks;
852 * Cleanup possible dangling ends...
854 static __cpuinit void smp_cleanup_boot(void)
857 * Paranoid: Set warm reset code and vector here back
863 * Reset trampoline flag
865 *((volatile int *) phys_to_virt(0x467)) = 0;
867 #ifndef CONFIG_HOTPLUG_CPU
869 * Free pages reserved for SMP bootup.
870 * When you add hotplug CPU support later remove this
871 * Note there is more work to be done for later CPU bootup.
874 free_page((unsigned long) __va(PAGE_SIZE));
875 free_page((unsigned long) __va(SMP_TRAMPOLINE_BASE));
880 * Fall back to non SMP mode after errors.
882 * RED-PEN audit/test this more. I bet there is more state messed up here.
884 static __init void disable_smp(void)
886 cpu_present_map = cpumask_of_cpu(0);
887 cpu_possible_map = cpumask_of_cpu(0);
888 if (smp_found_config)
889 phys_cpu_present_map = physid_mask_of_physid(boot_cpu_id);
891 phys_cpu_present_map = physid_mask_of_physid(0);
892 cpu_set(0, cpu_sibling_map[0]);
893 cpu_set(0, cpu_core_map[0]);
896 #ifdef CONFIG_HOTPLUG_CPU
898 * cpu_possible_map should be static, it cannot change as cpu's
899 * are onlined, or offlined. The reason is per-cpu data-structures
900 * are allocated by some modules at init time, and dont expect to
901 * do this dynamically on cpu arrival/departure.
902 * cpu_present_map on the other hand can change dynamically.
903 * In case when cpu_hotplug is not compiled, then we resort to current
904 * behaviour, which is cpu_possible == cpu_present.
905 * If cpu-hotplug is supported, then we need to preallocate for all
906 * those NR_CPUS, hence cpu_possible_map represents entire NR_CPUS range.
909 static void prefill_possible_map(void)
912 for (i = 0; i < NR_CPUS; i++)
913 cpu_set(i, cpu_possible_map);
918 * Various sanity checks.
920 static int __init smp_sanity_check(unsigned max_cpus)
922 if (!physid_isset(hard_smp_processor_id(), phys_cpu_present_map)) {
923 printk("weird, boot CPU (#%d) not listed by the BIOS.\n",
924 hard_smp_processor_id());
925 physid_set(hard_smp_processor_id(), phys_cpu_present_map);
929 * If we couldn't find an SMP configuration at boot time,
930 * get out of here now!
932 if (!smp_found_config) {
933 printk(KERN_NOTICE "SMP motherboard not detected.\n");
935 if (APIC_init_uniprocessor())
936 printk(KERN_NOTICE "Local APIC not detected."
937 " Using dummy APIC emulation.\n");
942 * Should not be necessary because the MP table should list the boot
943 * CPU too, but we do it for the sake of robustness anyway.
945 if (!physid_isset(boot_cpu_id, phys_cpu_present_map)) {
946 printk(KERN_NOTICE "weird, boot CPU (#%d) not listed by the BIOS.\n",
948 physid_set(hard_smp_processor_id(), phys_cpu_present_map);
952 * If we couldn't find a local APIC, then get out of here now!
954 if (APIC_INTEGRATED(apic_version[boot_cpu_id]) && !cpu_has_apic) {
955 printk(KERN_ERR "BIOS bug, local APIC #%d not detected!...\n",
957 printk(KERN_ERR "... forcing use of dummy APIC emulation. (tell your hw vendor)\n");
963 * If SMP should be disabled, then really disable it!
966 printk(KERN_INFO "SMP mode deactivated, forcing use of dummy APIC emulation.\n");
975 * Prepare for SMP bootup. The MP table or ACPI has been read
976 * earlier. Just do some sanity checking here and enable APIC mode.
978 void __init smp_prepare_cpus(unsigned int max_cpus)
980 nmi_watchdog_default();
981 current_cpu_data = boot_cpu_data;
982 current_thread_info()->cpu = 0; /* needed? */
984 #ifdef CONFIG_HOTPLUG_CPU
985 prefill_possible_map();
988 if (smp_sanity_check(max_cpus) < 0) {
989 printk(KERN_INFO "SMP disabled\n");
996 * Switch from PIC to APIC mode.
1001 if (GET_APIC_ID(apic_read(APIC_ID)) != boot_cpu_id) {
1002 panic("Boot APIC ID in local APIC unexpected (%d vs %d)",
1003 GET_APIC_ID(apic_read(APIC_ID)), boot_cpu_id);
1004 /* Or can we switch back to PIC here? */
1008 * Now start the IO-APICs
1010 if (!skip_ioapic_setup && nr_ioapics)
1016 * Set up local APIC timer on boot CPU.
1019 setup_boot_APIC_clock();
1023 * Early setup to make printk work.
1025 void __init smp_prepare_boot_cpu(void)
1027 int me = smp_processor_id();
1028 cpu_set(me, cpu_online_map);
1029 cpu_set(me, cpu_callout_map);
1030 cpu_set(0, cpu_sibling_map[0]);
1031 cpu_set(0, cpu_core_map[0]);
1032 per_cpu(cpu_state, me) = CPU_ONLINE;
1036 * Entry point to boot a CPU.
1038 int __cpuinit __cpu_up(unsigned int cpu)
1041 int apicid = cpu_present_to_apicid(cpu);
1043 WARN_ON(irqs_disabled());
1045 Dprintk("++++++++++++++++++++=_---CPU UP %u\n", cpu);
1047 if (apicid == BAD_APICID || apicid == boot_cpu_id ||
1048 !physid_isset(apicid, phys_cpu_present_map)) {
1049 printk("__cpu_up: bad cpu %d\n", cpu);
1054 * Already booted CPU?
1056 if (cpu_isset(cpu, cpu_callin_map)) {
1057 Dprintk("do_boot_cpu %d Already started\n", cpu);
1061 per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
1063 err = do_boot_cpu(cpu, apicid);
1065 Dprintk("do_boot_cpu failed %d\n", err);
1069 /* Unleash the CPU! */
1070 Dprintk("waiting for cpu %d\n", cpu);
1072 while (!cpu_isset(cpu, cpu_online_map))
1080 * Finish the SMP boot.
1082 void __init smp_cpus_done(unsigned int max_cpus)
1084 #ifndef CONFIG_HOTPLUG_CPU
1089 #ifdef CONFIG_X86_IO_APIC
1090 setup_ioapic_dest();
1095 check_nmi_watchdog();
1098 #ifdef CONFIG_HOTPLUG_CPU
1100 static void remove_siblinginfo(int cpu)
1104 for_each_cpu_mask(sibling, cpu_sibling_map[cpu])
1105 cpu_clear(cpu, cpu_sibling_map[sibling]);
1106 for_each_cpu_mask(sibling, cpu_core_map[cpu])
1107 cpu_clear(cpu, cpu_core_map[sibling]);
1108 cpus_clear(cpu_sibling_map[cpu]);
1109 cpus_clear(cpu_core_map[cpu]);
1110 phys_proc_id[cpu] = BAD_APICID;
1111 cpu_core_id[cpu] = BAD_APICID;
1114 void remove_cpu_from_maps(void)
1116 int cpu = smp_processor_id();
1118 cpu_clear(cpu, cpu_callout_map);
1119 cpu_clear(cpu, cpu_callin_map);
1120 clear_bit(cpu, &cpu_initialized); /* was set by cpu_init() */
1123 int __cpu_disable(void)
1125 int cpu = smp_processor_id();
1128 * Perhaps use cpufreq to drop frequency, but that could go
1129 * into generic code.
1131 * We won't take down the boot processor on i386 due to some
1132 * interrupts only being able to be serviced by the BSP.
1133 * Especially so if we're not using an IOAPIC -zwane
1138 disable_APIC_timer();
1142 * Allow any queued timer interrupts to get serviced
1143 * This is only a temporary solution until we cleanup
1144 * fixup_irqs as we do for IA64.
1149 local_irq_disable();
1150 remove_siblinginfo(cpu);
1152 /* It's now safe to remove this processor from the online map */
1153 cpu_clear(cpu, cpu_online_map);
1154 remove_cpu_from_maps();
1155 fixup_irqs(cpu_online_map);
1159 void __cpu_die(unsigned int cpu)
1161 /* We don't do anything here: idle task is faking death itself. */
1164 for (i = 0; i < 10; i++) {
1165 /* They ack this in play_dead by setting CPU_DEAD */
1166 if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
1167 printk ("CPU %d is now offline\n", cpu);
1172 printk(KERN_ERR "CPU %u didn't die...\n", cpu);
1175 #else /* ... !CONFIG_HOTPLUG_CPU */
1177 int __cpu_disable(void)
1182 void __cpu_die(unsigned int cpu)
1184 /* We said "no" in __cpu_disable */
1187 #endif /* CONFIG_HOTPLUG_CPU */