arch/s390/kernel/smp.c

   1 /*
   2  *  arch/s390/kernel/smp.c
   3  *
   4  *    Copyright IBM Corp. 1999, 2009
   5  *    Author(s): Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
   6  *               Martin Schwidefsky (schwidefsky@de.ibm.com)
   7  *               Heiko Carstens (heiko.carstens@de.ibm.com)
   8  *
   9  *  based on other smp stuff by
  10  *    (c) 1995 Alan Cox, CymruNET Ltd  <alan@cymru.net>
  11  *    (c) 1998 Ingo Molnar
  12  *
  13  * We work with logical cpu numbering everywhere we can. The only
  14  * functions using the real cpu address (got from STAP) are the sigp
  15  * functions. For all other functions we use the identity mapping.
  16  * That means that cpu_number_map[i] == i for every cpu. cpu_number_map is
  17  * used e.g. to find the idle task belonging to a logical cpu. Every array
  18  * in the kernel is sorted by the logical cpu number and not by the physical
  19  * one which is causing all the confusion with __cpu_logical_map and
  20  * cpu_number_map in other architectures.
  21  */
  22
  23 #define KMSG_COMPONENT "cpu"
  24 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  25
  26 #include <linux/workqueue.h>
  27 #include <linux/module.h>
  28 #include <linux/init.h>
  29 #include <linux/mm.h>
  30 #include <linux/err.h>
  31 #include <linux/spinlock.h>
  32 #include <linux/kernel_stat.h>
  33 #include <linux/delay.h>
  34 #include <linux/cache.h>
  35 #include <linux/interrupt.h>
  36 #include <linux/irqflags.h>
  37 #include <linux/cpu.h>
  38 #include <linux/timex.h>
  39 #include <linux/bootmem.h>
  40 #include <linux/slab.h>
  41 #include <linux/crash_dump.h>
  42 #include <asm/asm-offsets.h>
  43 #include <asm/ipl.h>
  44 #include <asm/setup.h>
  45 #include <asm/sigp.h>
  46 #include <asm/pgalloc.h>
  47 #include <asm/irq.h>
  48 #include <asm/cpcmd.h>
  49 #include <asm/tlbflush.h>
  50 #include <asm/timer.h>
  51 #include <asm/lowcore.h>
  52 #include <asm/sclp.h>
  53 #include <asm/cputime.h>
  54 #include <asm/vdso.h>
  55 #include <asm/cpu.h>
  56 #include "entry.h"
  57
  58 /* logical cpu to cpu address */
  59 unsigned short __cpu_logical_map[NR_CPUS];
  60
  61 static struct task_struct *current_set[NR_CPUS];
  62
  63 static u8 smp_cpu_type;
  64 static int smp_use_sigp_detection;
  65
  66 enum s390_cpu_state {
  67         CPU_STATE_STANDBY,
  68         CPU_STATE_CONFIGURED,
  69 };
  70
  71 DEFINE_MUTEX(smp_cpu_state_mutex);
  72 int smp_cpu_polarization[NR_CPUS];
  73 static int smp_cpu_state[NR_CPUS];
  74 static int cpu_management;
  75
  76 static DEFINE_PER_CPU(struct cpu, cpu_devices);
  77
  78 static void smp_ext_bitcall(int, int);
  79
  80 static int raw_cpu_stopped(int cpu)
  81 {
  82         u32 status;
  83
  84         switch (raw_sigp_ps(&status, 0, cpu, sigp_sense)) {
  85         case sigp_status_stored:
  86                 /* Check for stopped and check stop state */
  87                 if (status & 0x50)
  88                         return 1;
  89                 break;
  90         default:
  91                 break;
  92         }
  93         return 0;
  94 }
  95
  96 static inline int cpu_stopped(int cpu)
  97 {
  98         return raw_cpu_stopped(cpu_logical_map(cpu));
  99 }
 100
 101 /*
 102  * Ensure that PSW restart is done on an online CPU
 103  */
 104 void smp_restart_with_online_cpu(void)
 105 {
 106         int cpu;
 107
 108         for_each_online_cpu(cpu) {
 109                 if (stap() == __cpu_logical_map[cpu]) {
 110                         /* We are online: Enable DAT again and return */
 111                         __load_psw_mask(psw_kernel_bits | PSW_MASK_DAT);
 112                         return;
 113                 }
 114         }
 115         /* We are not online: Do PSW restart on an online CPU */
 116         while (sigp(cpu, sigp_restart) == sigp_busy)
 117                 cpu_relax();
 118         /* And stop ourself */
 119         while (raw_sigp(stap(), sigp_stop) == sigp_busy)
 120                 cpu_relax();
 121         for (;;);
 122 }
 123
 124 void smp_switch_to_ipl_cpu(void (*func)(void *), void *data)
 125 {
 126         struct _lowcore *lc, *current_lc;
 127         struct stack_frame *sf;
 128         struct pt_regs *regs;
 129         unsigned long sp;
 130
 131         if (smp_processor_id() == 0)
 132                 func(data);
 133         __load_psw_mask(PSW_DEFAULT_KEY | PSW_MASK_BASE |
 134                         PSW_MASK_EA | PSW_MASK_BA);
 135         /* Disable lowcore protection */
 136         __ctl_clear_bit(0, 28);
 137         current_lc = lowcore_ptr[smp_processor_id()];
 138         lc = lowcore_ptr[0];
 139         if (!lc)
 140                 lc = current_lc;
 141         lc->restart_psw.mask =
 142                 PSW_DEFAULT_KEY | PSW_MASK_BASE | PSW_MASK_EA | PSW_MASK_BA;
 143         lc->restart_psw.addr = PSW_ADDR_AMODE | (unsigned long) smp_restart_cpu;
 144         if (!cpu_online(0))
 145                 smp_switch_to_cpu(func, data, 0, stap(), __cpu_logical_map[0]);
 146         while (sigp(0, sigp_stop_and_store_status) == sigp_busy)
 147                 cpu_relax();
 148         sp = lc->panic_stack;
 149         sp -= sizeof(struct pt_regs);
 150         regs = (struct pt_regs *) sp;
 151         memcpy(&regs->gprs, &current_lc->gpregs_save_area, sizeof(regs->gprs));
 152         regs->psw = lc->psw_save_area;
 153         sp -= STACK_FRAME_OVERHEAD;
 154         sf = (struct stack_frame *) sp;
 155         sf->back_chain = regs->gprs[15];
 156         smp_switch_to_cpu(func, data, sp, stap(), __cpu_logical_map[0]);
 157 }
 158
 159 void smp_send_stop(void)
 160 {
 161         int cpu, rc;
 162
 163         /* Disable all interrupts/machine checks */
 164         __load_psw_mask(psw_kernel_bits | PSW_MASK_DAT);
 165         trace_hardirqs_off();
 166
 167         /* stop all processors */
 168         for_each_online_cpu(cpu) {
 169                 if (cpu == smp_processor_id())
 170                         continue;
 171                 do {
 172                         rc = sigp(cpu, sigp_stop);
 173                 } while (rc == sigp_busy);
 174
 175                 while (!cpu_stopped(cpu))
 176                         cpu_relax();
 177         }
 178 }
 179
 180 /*
 181  * This is the main routine where commands issued by other
 182  * cpus are handled.
 183  */
 184
 185 static void do_ext_call_interrupt(unsigned int ext_int_code,
 186                                   unsigned int param32, unsigned long param64)
 187 {
 188         unsigned long bits;
 189
 190         if (ext_int_code == 0x1202)
 191                 kstat_cpu(smp_processor_id()).irqs[EXTINT_EXC]++;
 192         else
 193                 kstat_cpu(smp_processor_id()).irqs[EXTINT_EMS]++;
 194         /*
 195          * handle bit signal external calls
 196          */
 197         bits = xchg(&S390_lowcore.ext_call_fast, 0);
 198
 199         if (test_bit(ec_schedule, &bits))
 200                 scheduler_ipi();
 201
 202         if (test_bit(ec_call_function, &bits))
 203                 generic_smp_call_function_interrupt();
 204
 205         if (test_bit(ec_call_function_single, &bits))
 206                 generic_smp_call_function_single_interrupt();
 207 }
 208
 209 /*
 210  * Send an external call sigp to another cpu and return without waiting
 211  * for its completion.
 212  */
 213 static void smp_ext_bitcall(int cpu, int sig)
 214 {
 215         int order;
 216
 217         /*
 218          * Set signaling bit in lowcore of target cpu and kick it
 219          */
 220         set_bit(sig, (unsigned long *) &lowcore_ptr[cpu]->ext_call_fast);
 221         while (1) {
 222                 order = smp_vcpu_scheduled(cpu) ?
 223                         sigp_external_call : sigp_emergency_signal;
 224                 if (sigp(cpu, order) != sigp_busy)
 225                         break;
 226                 udelay(10);
 227         }
 228 }
 229
 230 void arch_send_call_function_ipi_mask(const struct cpumask *mask)
 231 {
 232         int cpu;
 233
 234         for_each_cpu(cpu, mask)
 235                 smp_ext_bitcall(cpu, ec_call_function);
 236 }
 237
 238 void arch_send_call_function_single_ipi(int cpu)
 239 {
 240         smp_ext_bitcall(cpu, ec_call_function_single);
 241 }
 242
 243 #ifndef CONFIG_64BIT
 244 /*
 245  * this function sends a 'purge tlb' signal to another CPU.
 246  */
 247 static void smp_ptlb_callback(void *info)
 248 {
 249         __tlb_flush_local();
 250 }
 251
 252 void smp_ptlb_all(void)
 253 {
 254         on_each_cpu(smp_ptlb_callback, NULL, 1);
 255 }
 256 EXPORT_SYMBOL(smp_ptlb_all);
 257 #endif /* ! CONFIG_64BIT */
 258
 259 /*
 260  * this function sends a 'reschedule' IPI to another CPU.
 261  * it goes straight through and wastes no time serializing
 262  * anything. Worst case is that we lose a reschedule ...
 263  */
 264 void smp_send_reschedule(int cpu)
 265 {
 266         smp_ext_bitcall(cpu, ec_schedule);
 267 }
 268
 269 /*
 270  * parameter area for the set/clear control bit callbacks
 271  */
 272 struct ec_creg_mask_parms {
 273         unsigned long orvals[16];
 274         unsigned long andvals[16];
 275 };
 276
 277 /*
 278  * callback for setting/clearing control bits
 279  */
 280 static void smp_ctl_bit_callback(void *info)
 281 {
 282         struct ec_creg_mask_parms *pp = info;
 283         unsigned long cregs[16];
 284         int i;
 285
 286         __ctl_store(cregs, 0, 15);
 287         for (i = 0; i <= 15; i++)
 288                 cregs[i] = (cregs[i] & pp->andvals[i]) | pp->orvals[i];
 289         __ctl_load(cregs, 0, 15);
 290 }
 291
 292 /*
 293  * Set a bit in a control register of all cpus
 294  */
 295 void smp_ctl_set_bit(int cr, int bit)
 296 {
 297         struct ec_creg_mask_parms parms;
 298
 299         memset(&parms.orvals, 0, sizeof(parms.orvals));
 300         memset(&parms.andvals, 0xff, sizeof(parms.andvals));
 301         parms.orvals[cr] = 1UL << bit;
 302         on_each_cpu(smp_ctl_bit_callback, &parms, 1);
 303 }
 304 EXPORT_SYMBOL(smp_ctl_set_bit);
 305
 306 /*
 307  * Clear a bit in a control register of all cpus
 308  */
 309 void smp_ctl_clear_bit(int cr, int bit)
 310 {
 311         struct ec_creg_mask_parms parms;
 312
 313         memset(&parms.orvals, 0, sizeof(parms.orvals));
 314         memset(&parms.andvals, 0xff, sizeof(parms.andvals));
 315         parms.andvals[cr] = ~(1UL << bit);
 316         on_each_cpu(smp_ctl_bit_callback, &parms, 1);
 317 }
 318 EXPORT_SYMBOL(smp_ctl_clear_bit);
 319
 320 #if defined(CONFIG_ZFCPDUMP) || defined(CONFIG_CRASH_DUMP)
 321
 322 static void __init smp_get_save_area(unsigned int cpu, unsigned int phy_cpu)
 323 {
 324         if (ipl_info.type != IPL_TYPE_FCP_DUMP && !OLDMEM_BASE)
 325                 return;
 326         if (is_kdump_kernel())
 327                 return;
 328         if (cpu >= NR_CPUS) {
 329                 pr_warning("CPU %i exceeds the maximum %i and is excluded from "
 330                            "the dump\n", cpu, NR_CPUS - 1);
 331                 return;
 332         }
 333         zfcpdump_save_areas[cpu] = kmalloc(sizeof(struct save_area), GFP_KERNEL);
 334         while (raw_sigp(phy_cpu, sigp_stop_and_store_status) == sigp_busy)
 335                 cpu_relax();
 336         memcpy_real(zfcpdump_save_areas[cpu],
 337                     (void *)(unsigned long) store_prefix() + SAVE_AREA_BASE,
 338                     sizeof(struct save_area));
 339 }
 340
 341 struct save_area *zfcpdump_save_areas[NR_CPUS + 1];
 342 EXPORT_SYMBOL_GPL(zfcpdump_save_areas);
 343
 344 #else
 345
 346 static inline void smp_get_save_area(unsigned int cpu, unsigned int phy_cpu) { }
 347
 348 #endif /* CONFIG_ZFCPDUMP */
 349
 350 static int cpu_known(int cpu_id)
 351 {
 352         int cpu;
 353
 354         for_each_present_cpu(cpu) {
 355                 if (__cpu_logical_map[cpu] == cpu_id)
 356                         return 1;
 357         }
 358         return 0;
 359 }
 360
 361 static int smp_rescan_cpus_sigp(cpumask_t avail)
 362 {
 363         int cpu_id, logical_cpu;
 364
 365         logical_cpu = cpumask_first(&avail);
 366         if (logical_cpu >= nr_cpu_ids)
 367                 return 0;
 368         for (cpu_id = 0; cpu_id <= MAX_CPU_ADDRESS; cpu_id++) {
 369                 if (cpu_known(cpu_id))
 370                         continue;
 371                 __cpu_logical_map[logical_cpu] = cpu_id;
 372                 smp_cpu_polarization[logical_cpu] = POLARIZATION_UNKNWN;
 373                 if (!cpu_stopped(logical_cpu))
 374                         continue;
 375                 set_cpu_present(logical_cpu, true);
 376                 smp_cpu_state[logical_cpu] = CPU_STATE_CONFIGURED;
 377                 logical_cpu = cpumask_next(logical_cpu, &avail);
 378                 if (logical_cpu >= nr_cpu_ids)
 379                         break;
 380         }
 381         return 0;
 382 }
 383
 384 static int smp_rescan_cpus_sclp(cpumask_t avail)
 385 {
 386         struct sclp_cpu_info *info;
 387         int cpu_id, logical_cpu, cpu;
 388         int rc;
 389
 390         logical_cpu = cpumask_first(&avail);
 391         if (logical_cpu >= nr_cpu_ids)
 392                 return 0;
 393         info = kmalloc(sizeof(*info), GFP_KERNEL);
 394         if (!info)
 395                 return -ENOMEM;
 396         rc = sclp_get_cpu_info(info);
 397         if (rc)
 398                 goto out;
 399         for (cpu = 0; cpu < info->combined; cpu++) {
 400                 if (info->has_cpu_type && info->cpu[cpu].type != smp_cpu_type)
 401                         continue;
 402                 cpu_id = info->cpu[cpu].address;
 403                 if (cpu_known(cpu_id))
 404                         continue;
 405                 __cpu_logical_map[logical_cpu] = cpu_id;
 406                 smp_cpu_polarization[logical_cpu] = POLARIZATION_UNKNWN;
 407                 set_cpu_present(logical_cpu, true);
 408                 if (cpu >= info->configured)
 409                         smp_cpu_state[logical_cpu] = CPU_STATE_STANDBY;
 410                 else
 411                         smp_cpu_state[logical_cpu] = CPU_STATE_CONFIGURED;
 412                 logical_cpu = cpumask_next(logical_cpu, &avail);
 413                 if (logical_cpu >= nr_cpu_ids)
 414                         break;
 415         }
 416 out:
 417         kfree(info);
 418         return rc;
 419 }
 420
 421 static int __smp_rescan_cpus(void)
 422 {
 423         cpumask_t avail;
 424
 425         cpumask_xor(&avail, cpu_possible_mask, cpu_present_mask);
 426         if (smp_use_sigp_detection)
 427                 return smp_rescan_cpus_sigp(avail);
 428         else
 429                 return smp_rescan_cpus_sclp(avail);
 430 }
 431
 432 static void __init smp_detect_cpus(void)
 433 {
 434         unsigned int cpu, c_cpus, s_cpus;
 435         struct sclp_cpu_info *info;
 436         u16 boot_cpu_addr, cpu_addr;
 437
 438         c_cpus = 1;
 439         s_cpus = 0;
 440         boot_cpu_addr = __cpu_logical_map[0];
 441         info = kmalloc(sizeof(*info), GFP_KERNEL);
 442         if (!info)
 443                 panic("smp_detect_cpus failed to allocate memory\n");
 444 #ifdef CONFIG_CRASH_DUMP
 445         if (OLDMEM_BASE && !is_kdump_kernel()) {
 446                 struct save_area *save_area;
 447
 448                 save_area = kmalloc(sizeof(*save_area), GFP_KERNEL);
 449                 if (!save_area)
 450                         panic("could not allocate memory for save area\n");
 451                 copy_oldmem_page(1, (void *) save_area, sizeof(*save_area),
 452                                  0x200, 0);
 453                 zfcpdump_save_areas[0] = save_area;
 454         }
 455 #endif
 456         /* Use sigp detection algorithm if sclp doesn't work. */
 457         if (sclp_get_cpu_info(info)) {
 458                 smp_use_sigp_detection = 1;
 459                 for (cpu = 0; cpu <= MAX_CPU_ADDRESS; cpu++) {
 460                         if (cpu == boot_cpu_addr)
 461                                 continue;
 462                         if (!raw_cpu_stopped(cpu))
 463                                 continue;
 464                         smp_get_save_area(c_cpus, cpu);
 465                         c_cpus++;
 466                 }
 467                 goto out;
 468         }
 469
 470         if (info->has_cpu_type) {
 471                 for (cpu = 0; cpu < info->combined; cpu++) {
 472                         if (info->cpu[cpu].address == boot_cpu_addr) {
 473                                 smp_cpu_type = info->cpu[cpu].type;
 474                                 break;
 475                         }
 476                 }
 477         }
 478
 479         for (cpu = 0; cpu < info->combined; cpu++) {
 480                 if (info->has_cpu_type && info->cpu[cpu].type != smp_cpu_type)
 481                         continue;
 482                 cpu_addr = info->cpu[cpu].address;
 483                 if (cpu_addr == boot_cpu_addr)
 484                         continue;
 485                 if (!raw_cpu_stopped(cpu_addr)) {
 486                         s_cpus++;
 487                         continue;
 488                 }
 489                 smp_get_save_area(c_cpus, cpu_addr);
 490                 c_cpus++;
 491         }
 492 out:
 493         kfree(info);
 494         pr_info("%d configured CPUs, %d standby CPUs\n", c_cpus, s_cpus);
 495         get_online_cpus();
 496         __smp_rescan_cpus();
 497         put_online_cpus();
 498 }
 499
 500 /*
 501  *      Activate a secondary processor.
 502  */
 503 int __cpuinit start_secondary(void *cpuvoid)
 504 {
 505         cpu_init();
 506         preempt_disable();
 507         init_cpu_timer();
 508         init_cpu_vtimer();
 509         pfault_init();
 510
 511         notify_cpu_starting(smp_processor_id());
 512         ipi_call_lock();
 513         set_cpu_online(smp_processor_id(), true);
 514         ipi_call_unlock();
 515         __ctl_clear_bit(0, 28); /* Disable lowcore protection */
 516         S390_lowcore.restart_psw.mask =
 517                 PSW_DEFAULT_KEY | PSW_MASK_BASE | PSW_MASK_EA | PSW_MASK_BA;
 518         S390_lowcore.restart_psw.addr =
 519                 PSW_ADDR_AMODE | (unsigned long) psw_restart_int_handler;
 520         __ctl_set_bit(0, 28); /* Enable lowcore protection */
 521         /*
 522          * Wait until the cpu which brought this one up marked it
 523          * active before enabling interrupts.
 524          */
 525         while (!cpumask_test_cpu(smp_processor_id(), cpu_active_mask))
 526                 cpu_relax();
 527         local_irq_enable();
 528         /* cpu_idle will call schedule for us */
 529         cpu_idle();
 530         return 0;
 531 }
 532
 533 struct create_idle {
 534         struct work_struct work;
 535         struct task_struct *idle;
 536         struct completion done;
 537         int cpu;
 538 };
 539
 540 static void __cpuinit smp_fork_idle(struct work_struct *work)
 541 {
 542         struct create_idle *c_idle;
 543
 544         c_idle = container_of(work, struct create_idle, work);
 545         c_idle->idle = fork_idle(c_idle->cpu);
 546         complete(&c_idle->done);
 547 }
 548
 549 static int __cpuinit smp_alloc_lowcore(int cpu)
 550 {
 551         unsigned long async_stack, panic_stack;
 552         struct _lowcore *lowcore;
 553
 554         lowcore = (void *) __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
 555         if (!lowcore)
 556                 return -ENOMEM;
 557         async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER);
 558         panic_stack = __get_free_page(GFP_KERNEL);
 559         if (!panic_stack || !async_stack)
 560                 goto out;
 561         memcpy(lowcore, &S390_lowcore, 512);
 562         memset((char *)lowcore + 512, 0, sizeof(*lowcore) - 512);
 563         lowcore->async_stack = async_stack + ASYNC_SIZE;
 564         lowcore->panic_stack = panic_stack + PAGE_SIZE;
 565         lowcore->restart_psw.mask =
 566                 PSW_DEFAULT_KEY | PSW_MASK_BASE | PSW_MASK_EA | PSW_MASK_BA;
 567         lowcore->restart_psw.addr =
 568                 PSW_ADDR_AMODE | (unsigned long) restart_int_handler;
 569         if (user_mode != HOME_SPACE_MODE)
 570                 lowcore->restart_psw.mask |= PSW_ASC_HOME;
 571 #ifndef CONFIG_64BIT
 572         if (MACHINE_HAS_IEEE) {
 573                 unsigned long save_area;
 574
 575                 save_area = get_zeroed_page(GFP_KERNEL);
 576                 if (!save_area)
 577                         goto out;
 578                 lowcore->extended_save_area_addr = (u32) save_area;
 579         }
 580 #else
 581         if (vdso_alloc_per_cpu(cpu, lowcore))
 582                 goto out;
 583 #endif
 584         lowcore_ptr[cpu] = lowcore;
 585         return 0;
 586
 587 out:
 588         free_page(panic_stack);
 589         free_pages(async_stack, ASYNC_ORDER);
 590         free_pages((unsigned long) lowcore, LC_ORDER);
 591         return -ENOMEM;
 592 }
 593
 594 static void smp_free_lowcore(int cpu)
 595 {
 596         struct _lowcore *lowcore;
 597
 598         lowcore = lowcore_ptr[cpu];
 599 #ifndef CONFIG_64BIT
 600         if (MACHINE_HAS_IEEE)
 601                 free_page((unsigned long) lowcore->extended_save_area_addr);
 602 #else
 603         vdso_free_per_cpu(cpu, lowcore);
 604 #endif
 605         free_page(lowcore->panic_stack - PAGE_SIZE);
 606         free_pages(lowcore->async_stack - ASYNC_SIZE, ASYNC_ORDER);
 607         free_pages((unsigned long) lowcore, LC_ORDER);
 608         lowcore_ptr[cpu] = NULL;
 609 }
 610
 611 /* Upping and downing of CPUs */
 612 int __cpuinit __cpu_up(unsigned int cpu)
 613 {
 614         struct _lowcore *cpu_lowcore;
 615         struct create_idle c_idle;
 616         struct task_struct *idle;
 617         struct stack_frame *sf;
 618         u32 lowcore;
 619         int ccode;
 620
 621         if (smp_cpu_state[cpu] != CPU_STATE_CONFIGURED)
 622                 return -EIO;
 623         idle = current_set[cpu];
 624         if (!idle) {
 625                 c_idle.done = COMPLETION_INITIALIZER_ONSTACK(c_idle.done);
 626                 INIT_WORK_ONSTACK(&c_idle.work, smp_fork_idle);
 627                 c_idle.cpu = cpu;
 628                 schedule_work(&c_idle.work);
 629                 wait_for_completion(&c_idle.done);
 630                 if (IS_ERR(c_idle.idle))
 631                         return PTR_ERR(c_idle.idle);
 632                 idle = c_idle.idle;
 633                 current_set[cpu] = c_idle.idle;
 634         }
 635         init_idle(idle, cpu);
 636         if (smp_alloc_lowcore(cpu))
 637                 return -ENOMEM;
 638         do {
 639                 ccode = sigp(cpu, sigp_initial_cpu_reset);
 640                 if (ccode == sigp_busy)
 641                         udelay(10);
 642                 if (ccode == sigp_not_operational)
 643                         goto err_out;
 644         } while (ccode == sigp_busy);
 645
 646         lowcore = (u32)(unsigned long)lowcore_ptr[cpu];
 647         while (sigp_p(lowcore, cpu, sigp_set_prefix) == sigp_busy)
 648                 udelay(10);
 649
 650         cpu_lowcore = lowcore_ptr[cpu];
 651         cpu_lowcore->kernel_stack = (unsigned long)
 652                 task_stack_page(idle) + THREAD_SIZE;
 653         cpu_lowcore->thread_info = (unsigned long) task_thread_info(idle);
 654         sf = (struct stack_frame *) (cpu_lowcore->kernel_stack
 655                                      - sizeof(struct pt_regs)
 656                                      - sizeof(struct stack_frame));
 657         memset(sf, 0, sizeof(struct stack_frame));
 658         sf->gprs[9] = (unsigned long) sf;
 659         cpu_lowcore->save_area[15] = (unsigned long) sf;
 660         __ctl_store(cpu_lowcore->cregs_save_area, 0, 15);
 661         atomic_inc(&init_mm.context.attach_count);
 662         asm volatile(
 663                 "       stam    0,15,0(%0)"
 664                 : : "a" (&cpu_lowcore->access_regs_save_area) : "memory");
 665         cpu_lowcore->percpu_offset = __per_cpu_offset[cpu];
 666         cpu_lowcore->current_task = (unsigned long) idle;
 667         cpu_lowcore->cpu_nr = cpu;
 668         cpu_lowcore->kernel_asce = S390_lowcore.kernel_asce;
 669         cpu_lowcore->machine_flags = S390_lowcore.machine_flags;
 670         cpu_lowcore->ftrace_func = S390_lowcore.ftrace_func;
 671         memcpy(cpu_lowcore->stfle_fac_list, S390_lowcore.stfle_fac_list,
 672                MAX_FACILITY_BIT/8);
 673         eieio();
 674
 675         while (sigp(cpu, sigp_restart) == sigp_busy)
 676                 udelay(10);
 677
 678         while (!cpu_online(cpu))
 679                 cpu_relax();
 680         return 0;
 681
 682 err_out:
 683         smp_free_lowcore(cpu);
 684         return -EIO;
 685 }
 686
 687 static int __init setup_possible_cpus(char *s)
 688 {
 689         int pcpus, cpu;
 690
 691         pcpus = simple_strtoul(s, NULL, 0);
 692         init_cpu_possible(cpumask_of(0));
 693         for (cpu = 1; cpu < pcpus && cpu < nr_cpu_ids; cpu++)
 694                 set_cpu_possible(cpu, true);
 695         return 0;
 696 }
 697 early_param("possible_cpus", setup_possible_cpus);
 698
 699 #ifdef CONFIG_HOTPLUG_CPU
 700
 701 int __cpu_disable(void)
 702 {
 703         struct ec_creg_mask_parms cr_parms;
 704         int cpu = smp_processor_id();
 705
 706         set_cpu_online(cpu, false);
 707
 708         /* Disable pfault pseudo page faults on this cpu. */
 709         pfault_fini();
 710
 711         memset(&cr_parms.orvals, 0, sizeof(cr_parms.orvals));
 712         memset(&cr_parms.andvals, 0xff, sizeof(cr_parms.andvals));
 713
 714         /* disable all external interrupts */
 715         cr_parms.orvals[0] = 0;
 716         cr_parms.andvals[0] = ~(1 << 15 | 1 << 14 | 1 << 13 | 1 << 11 |
 717                                 1 << 10 | 1 <<  9 | 1 <<  6 | 1 <<  5 |
 718                                 1 <<  4);
 719         /* disable all I/O interrupts */
 720         cr_parms.orvals[6] = 0;
 721         cr_parms.andvals[6] = ~(1 << 31 | 1 << 30 | 1 << 29 | 1 << 28 |
 722                                 1 << 27 | 1 << 26 | 1 << 25 | 1 << 24);
 723         /* disable most machine checks */
 724         cr_parms.orvals[14] = 0;
 725         cr_parms.andvals[14] = ~(1 << 28 | 1 << 27 | 1 << 26 |
 726                                  1 << 25 | 1 << 24);
 727
 728         smp_ctl_bit_callback(&cr_parms);
 729
 730         return 0;
 731 }
 732
 733 void __cpu_die(unsigned int cpu)
 734 {
 735         /* Wait until target cpu is down */
 736         while (!cpu_stopped(cpu))
 737                 cpu_relax();
 738         while (sigp_p(0, cpu, sigp_set_prefix) == sigp_busy)
 739                 udelay(10);
 740         smp_free_lowcore(cpu);
 741         atomic_dec(&init_mm.context.attach_count);
 742 }
 743
 744 void __noreturn cpu_die(void)
 745 {
 746         idle_task_exit();
 747         while (sigp(smp_processor_id(), sigp_stop) == sigp_busy)
 748                 cpu_relax();
 749         for (;;);
 750 }
 751
 752 #endif /* CONFIG_HOTPLUG_CPU */
 753
 754 void __init smp_prepare_cpus(unsigned int max_cpus)
 755 {
 756 #ifndef CONFIG_64BIT
 757         unsigned long save_area = 0;
 758 #endif
 759         unsigned long async_stack, panic_stack;
 760         struct _lowcore *lowcore;
 761
 762         smp_detect_cpus();
 763
 764         /* request the 0x1201 emergency signal external interrupt */
 765         if (register_external_interrupt(0x1201, do_ext_call_interrupt) != 0)
 766                 panic("Couldn't request external interrupt 0x1201");
 767         /* request the 0x1202 external call external interrupt */
 768         if (register_external_interrupt(0x1202, do_ext_call_interrupt) != 0)
 769                 panic("Couldn't request external interrupt 0x1202");
 770
 771         /* Reallocate current lowcore, but keep its contents. */
 772         lowcore = (void *) __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
 773         panic_stack = __get_free_page(GFP_KERNEL);
 774         async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER);
 775         BUG_ON(!lowcore || !panic_stack || !async_stack);
 776 #ifndef CONFIG_64BIT
 777         if (MACHINE_HAS_IEEE)
 778                 save_area = get_zeroed_page(GFP_KERNEL);
 779 #endif
 780         local_irq_disable();
 781         local_mcck_disable();
 782         lowcore_ptr[smp_processor_id()] = lowcore;
 783         *lowcore = S390_lowcore;
 784         lowcore->panic_stack = panic_stack + PAGE_SIZE;
 785         lowcore->async_stack = async_stack + ASYNC_SIZE;
 786 #ifndef CONFIG_64BIT
 787         if (MACHINE_HAS_IEEE)
 788                 lowcore->extended_save_area_addr = (u32) save_area;
 789 #endif
 790         set_prefix((u32)(unsigned long) lowcore);
 791         local_mcck_enable();
 792         local_irq_enable();
 793 #ifdef CONFIG_64BIT
 794         if (vdso_alloc_per_cpu(smp_processor_id(), &S390_lowcore))
 795                 BUG();
 796 #endif
 797 }
 798
 799 void __init smp_prepare_boot_cpu(void)
 800 {
 801         BUG_ON(smp_processor_id() != 0);
 802
 803         current_thread_info()->cpu = 0;
 804         set_cpu_present(0, true);
 805         set_cpu_online(0, true);
 806         S390_lowcore.percpu_offset = __per_cpu_offset[0];
 807         current_set[0] = current;
 808         smp_cpu_state[0] = CPU_STATE_CONFIGURED;
 809         smp_cpu_polarization[0] = POLARIZATION_UNKNWN;
 810 }
 811
 812 void __init smp_cpus_done(unsigned int max_cpus)
 813 {
 814 }
 815
 816 void __init smp_setup_processor_id(void)
 817 {
 818         S390_lowcore.cpu_nr = 0;
 819         __cpu_logical_map[0] = stap();
 820 }
 821
 822 /*
 823  * the frequency of the profiling timer can be changed
 824  * by writing a multiplier value into /proc/profile.
 825  *
 826  * usually you want to run this on all CPUs ;)
 827  */
 828 int setup_profiling_timer(unsigned int multiplier)
 829 {
 830         return 0;
 831 }
 832
 833 #ifdef CONFIG_HOTPLUG_CPU
 834 static ssize_t cpu_configure_show(struct sys_device *dev,
 835                                 struct sysdev_attribute *attr, char *buf)
 836 {
 837         ssize_t count;
 838
 839         mutex_lock(&smp_cpu_state_mutex);
 840         count = sprintf(buf, "%d\n", smp_cpu_state[dev->id]);
 841         mutex_unlock(&smp_cpu_state_mutex);
 842         return count;
 843 }
 844
 845 static ssize_t cpu_configure_store(struct sys_device *dev,
 846                                   struct sysdev_attribute *attr,
 847                                   const char *buf, size_t count)
 848 {
 849         int cpu = dev->id;
 850         int val, rc;
 851         char delim;
 852
 853         if (sscanf(buf, "%d %c", &val, &delim) != 1)
 854                 return -EINVAL;
 855         if (val != 0 && val != 1)
 856                 return -EINVAL;
 857
 858         get_online_cpus();
 859         mutex_lock(&smp_cpu_state_mutex);
 860         rc = -EBUSY;
 861         /* disallow configuration changes of online cpus and cpu 0 */
 862         if (cpu_online(cpu) || cpu == 0)
 863                 goto out;
 864         rc = 0;
 865         switch (val) {
 866         case 0:
 867                 if (smp_cpu_state[cpu] == CPU_STATE_CONFIGURED) {
 868                         rc = sclp_cpu_deconfigure(__cpu_logical_map[cpu]);
 869                         if (!rc) {
 870                                 smp_cpu_state[cpu] = CPU_STATE_STANDBY;
 871                                 smp_cpu_polarization[cpu] = POLARIZATION_UNKNWN;
 872                         }
 873                 }
 874                 break;
 875         case 1:
 876                 if (smp_cpu_state[cpu] == CPU_STATE_STANDBY) {
 877                         rc = sclp_cpu_configure(__cpu_logical_map[cpu]);
 878                         if (!rc) {
 879                                 smp_cpu_state[cpu] = CPU_STATE_CONFIGURED;
 880                                 smp_cpu_polarization[cpu] = POLARIZATION_UNKNWN;
 881                         }
 882                 }
 883                 break;
 884         default:
 885                 break;
 886         }
 887 out:
 888         mutex_unlock(&smp_cpu_state_mutex);
 889         put_online_cpus();
 890         return rc ? rc : count;
 891 }
 892 static SYSDEV_ATTR(configure, 0644, cpu_configure_show, cpu_configure_store);
 893 #endif /* CONFIG_HOTPLUG_CPU */
 894
 895 static ssize_t cpu_polarization_show(struct sys_device *dev,
 896                                      struct sysdev_attribute *attr, char *buf)
 897 {
 898         int cpu = dev->id;
 899         ssize_t count;
 900
 901         mutex_lock(&smp_cpu_state_mutex);
 902         switch (smp_cpu_polarization[cpu]) {
 903         case POLARIZATION_HRZ:
 904                 count = sprintf(buf, "horizontal\n");
 905                 break;
 906         case POLARIZATION_VL:
 907                 count = sprintf(buf, "vertical:low\n");
 908                 break;
 909         case POLARIZATION_VM:
 910                 count = sprintf(buf, "vertical:medium\n");
 911                 break;
 912         case POLARIZATION_VH:
 913                 count = sprintf(buf, "vertical:high\n");
 914                 break;
 915         default:
 916                 count = sprintf(buf, "unknown\n");
 917                 break;
 918         }
 919         mutex_unlock(&smp_cpu_state_mutex);
 920         return count;
 921 }
 922 static SYSDEV_ATTR(polarization, 0444, cpu_polarization_show, NULL);
 923
 924 static ssize_t show_cpu_address(struct sys_device *dev,
 925                                 struct sysdev_attribute *attr, char *buf)
 926 {
 927         return sprintf(buf, "%d\n", __cpu_logical_map[dev->id]);
 928 }
 929 static SYSDEV_ATTR(address, 0444, show_cpu_address, NULL);
 930
 931
 932 static struct attribute *cpu_common_attrs[] = {
 933 #ifdef CONFIG_HOTPLUG_CPU
 934         &attr_configure.attr,
 935 #endif
 936         &attr_address.attr,
 937         &attr_polarization.attr,
 938         NULL,
 939 };
 940
 941 static struct attribute_group cpu_common_attr_group = {
 942         .attrs = cpu_common_attrs,
 943 };
 944
 945 static ssize_t show_capability(struct sys_device *dev,
 946                                 struct sysdev_attribute *attr, char *buf)
 947 {
 948         unsigned int capability;
 949         int rc;
 950
 951         rc = get_cpu_capability(&capability);
 952         if (rc)
 953                 return rc;
 954         return sprintf(buf, "%u\n", capability);
 955 }
 956 static SYSDEV_ATTR(capability, 0444, show_capability, NULL);
 957
 958 static ssize_t show_idle_count(struct sys_device *dev,
 959                                 struct sysdev_attribute *attr, char *buf)
 960 {
 961         struct s390_idle_data *idle;
 962         unsigned long long idle_count;
 963         unsigned int sequence;
 964
 965         idle = &per_cpu(s390_idle, dev->id);
 966 repeat:
 967         sequence = idle->sequence;
 968         smp_rmb();
 969         if (sequence & 1)
 970                 goto repeat;
 971         idle_count = idle->idle_count;
 972         if (idle->idle_enter)
 973                 idle_count++;
 974         smp_rmb();
 975         if (idle->sequence != sequence)
 976                 goto repeat;
 977         return sprintf(buf, "%llu\n", idle_count);
 978 }
 979 static SYSDEV_ATTR(idle_count, 0444, show_idle_count, NULL);
 980
 981 static ssize_t show_idle_time(struct sys_device *dev,
 982                                 struct sysdev_attribute *attr, char *buf)
 983 {
 984         struct s390_idle_data *idle;
 985         unsigned long long now, idle_time, idle_enter;
 986         unsigned int sequence;
 987
 988         idle = &per_cpu(s390_idle, dev->id);
 989         now = get_clock();
 990 repeat:
 991         sequence = idle->sequence;
 992         smp_rmb();
 993         if (sequence & 1)
 994                 goto repeat;
 995         idle_time = idle->idle_time;
 996         idle_enter = idle->idle_enter;
 997         if (idle_enter != 0ULL && idle_enter < now)
 998                 idle_time += now - idle_enter;
 999         smp_rmb();
1000         if (idle->sequence != sequence)
1001                 goto repeat;
1002         return sprintf(buf, "%llu\n", idle_time >> 12);
1003 }
1004 static SYSDEV_ATTR(idle_time_us, 0444, show_idle_time, NULL);
1005
1006 static struct attribute *cpu_online_attrs[] = {
1007         &attr_capability.attr,
1008         &attr_idle_count.attr,
1009         &attr_idle_time_us.attr,
1010         NULL,
1011 };
1012
1013 static struct attribute_group cpu_online_attr_group = {
1014         .attrs = cpu_online_attrs,
1015 };
1016
1017 static int __cpuinit smp_cpu_notify(struct notifier_block *self,
1018                                     unsigned long action, void *hcpu)
1019 {
1020         unsigned int cpu = (unsigned int)(long)hcpu;
1021         struct cpu *c = &per_cpu(cpu_devices, cpu);
1022         struct sys_device *s = &c->sysdev;
1023         int err = 0;
1024
1025         switch (action) {
1026         case CPU_ONLINE:
1027         case CPU_ONLINE_FROZEN:
1028                 err = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
1029                 break;
1030         case CPU_DEAD:
1031         case CPU_DEAD_FROZEN:
1032                 sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
1033                 break;
1034         }
1035         return notifier_from_errno(err);
1036 }
1037
1038 static struct notifier_block __cpuinitdata smp_cpu_nb = {
1039         .notifier_call = smp_cpu_notify,
1040 };
1041
1042 static int __devinit smp_add_present_cpu(int cpu)
1043 {
1044         struct cpu *c = &per_cpu(cpu_devices, cpu);
1045         struct sys_device *s = &c->sysdev;
1046         int rc;
1047
1048         c->hotpluggable = 1;
1049         rc = register_cpu(c, cpu);
1050         if (rc)
1051                 goto out;
1052         rc = sysfs_create_group(&s->kobj, &cpu_common_attr_group);
1053         if (rc)
1054                 goto out_cpu;
1055         if (!cpu_online(cpu))
1056                 goto out;
1057         rc = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
1058         if (!rc)
1059                 return 0;
1060         sysfs_remove_group(&s->kobj, &cpu_common_attr_group);
1061 out_cpu:
1062 #ifdef CONFIG_HOTPLUG_CPU
1063         unregister_cpu(c);
1064 #endif
1065 out:
1066         return rc;
1067 }
1068
1069 #ifdef CONFIG_HOTPLUG_CPU
1070
1071 int __ref smp_rescan_cpus(void)
1072 {
1073         cpumask_t newcpus;
1074         int cpu;
1075         int rc;
1076
1077         get_online_cpus();
1078         mutex_lock(&smp_cpu_state_mutex);
1079         cpumask_copy(&newcpus, cpu_present_mask);
1080         rc = __smp_rescan_cpus();
1081         if (rc)
1082                 goto out;
1083         cpumask_andnot(&newcpus, cpu_present_mask, &newcpus);
1084         for_each_cpu(cpu, &newcpus) {
1085                 rc = smp_add_present_cpu(cpu);
1086                 if (rc)
1087                         set_cpu_present(cpu, false);
1088         }
1089         rc = 0;
1090 out:
1091         mutex_unlock(&smp_cpu_state_mutex);
1092         put_online_cpus();
1093         if (!cpumask_empty(&newcpus))
1094                 topology_schedule_update();
1095         return rc;
1096 }
1097
1098 static ssize_t __ref rescan_store(struct sysdev_class *class,
1099                                   struct sysdev_class_attribute *attr,
1100                                   const char *buf,
1101                                   size_t count)
1102 {
1103         int rc;
1104
1105         rc = smp_rescan_cpus();
1106         return rc ? rc : count;
1107 }
1108 static SYSDEV_CLASS_ATTR(rescan, 0200, NULL, rescan_store);
1109 #endif /* CONFIG_HOTPLUG_CPU */
1110
1111 static ssize_t dispatching_show(struct sysdev_class *class,
1112                                 struct sysdev_class_attribute *attr,
1113                                 char *buf)
1114 {
1115         ssize_t count;
1116
1117         mutex_lock(&smp_cpu_state_mutex);
1118         count = sprintf(buf, "%d\n", cpu_management);
1119         mutex_unlock(&smp_cpu_state_mutex);
1120         return count;
1121 }
1122
1123 static ssize_t dispatching_store(struct sysdev_class *dev,
1124                                  struct sysdev_class_attribute *attr,
1125                                  const char *buf,
1126                                  size_t count)
1127 {
1128         int val, rc;
1129         char delim;
1130
1131         if (sscanf(buf, "%d %c", &val, &delim) != 1)
1132                 return -EINVAL;
1133         if (val != 0 && val != 1)
1134                 return -EINVAL;
1135         rc = 0;
1136         get_online_cpus();
1137         mutex_lock(&smp_cpu_state_mutex);
1138         if (cpu_management == val)
1139                 goto out;
1140         rc = topology_set_cpu_management(val);
1141         if (!rc)
1142                 cpu_management = val;
1143 out:
1144         mutex_unlock(&smp_cpu_state_mutex);
1145         put_online_cpus();
1146         return rc ? rc : count;
1147 }
1148 static SYSDEV_CLASS_ATTR(dispatching, 0644, dispatching_show,
1149                          dispatching_store);
1150
1151 static int __init topology_init(void)
1152 {
1153         int cpu;
1154         int rc;
1155
1156         register_cpu_notifier(&smp_cpu_nb);
1157
1158 #ifdef CONFIG_HOTPLUG_CPU
1159         rc = sysdev_class_create_file(&cpu_sysdev_class, &attr_rescan);
1160         if (rc)
1161                 return rc;
1162 #endif
1163         rc = sysdev_class_create_file(&cpu_sysdev_class, &attr_dispatching);
1164         if (rc)
1165                 return rc;
1166         for_each_present_cpu(cpu) {
1167                 rc = smp_add_present_cpu(cpu);
1168                 if (rc)
1169                         return rc;
1170         }
1171         return 0;
1172 }
1173 subsys_initcall(topology_init);