arch/s390/kernel/smp.c

   1 /*
   2  *  arch/s390/kernel/smp.c
   3  *
   4  *    Copyright IBM Corp. 1999, 2009
   5  *    Author(s): Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
   6  *               Martin Schwidefsky (schwidefsky@de.ibm.com)
   7  *               Heiko Carstens (heiko.carstens@de.ibm.com)
   8  *
   9  *  based on other smp stuff by
  10  *    (c) 1995 Alan Cox, CymruNET Ltd  <alan@cymru.net>
  11  *    (c) 1998 Ingo Molnar
  12  *
  13  * We work with logical cpu numbering everywhere we can. The only
  14  * functions using the real cpu address (got from STAP) are the sigp
  15  * functions. For all other functions we use the identity mapping.
  16  * That means that cpu_number_map[i] == i for every cpu. cpu_number_map is
  17  * used e.g. to find the idle task belonging to a logical cpu. Every array
  18  * in the kernel is sorted by the logical cpu number and not by the physical
  19  * one which is causing all the confusion with __cpu_logical_map and
  20  * cpu_number_map in other architectures.
  21  */
  22
  23 #define KMSG_COMPONENT "cpu"
  24 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  25
  26 #include <linux/module.h>
  27 #include <linux/init.h>
  28 #include <linux/mm.h>
  29 #include <linux/err.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/kernel_stat.h>
  32 #include <linux/delay.h>
  33 #include <linux/cache.h>
  34 #include <linux/interrupt.h>
  35 #include <linux/irqflags.h>
  36 #include <linux/cpu.h>
  37 #include <linux/timex.h>
  38 #include <linux/bootmem.h>
  39 #include <asm/asm-offsets.h>
  40 #include <asm/ipl.h>
  41 #include <asm/setup.h>
  42 #include <asm/sigp.h>
  43 #include <asm/pgalloc.h>
  44 #include <asm/irq.h>
  45 #include <asm/s390_ext.h>
  46 #include <asm/cpcmd.h>
  47 #include <asm/tlbflush.h>
  48 #include <asm/timer.h>
  49 #include <asm/lowcore.h>
  50 #include <asm/sclp.h>
  51 #include <asm/cputime.h>
  52 #include <asm/vdso.h>
  53 #include <asm/cpu.h>
  54 #include "entry.h"
  55
  56 /* logical cpu to cpu address */
  57 unsigned short __cpu_logical_map[NR_CPUS];
  58
  59 static struct task_struct *current_set[NR_CPUS];
  60
  61 static u8 smp_cpu_type;
  62 static int smp_use_sigp_detection;
  63
  64 enum s390_cpu_state {
  65         CPU_STATE_STANDBY,
  66         CPU_STATE_CONFIGURED,
  67 };
  68
  69 DEFINE_MUTEX(smp_cpu_state_mutex);
  70 int smp_cpu_polarization[NR_CPUS];
  71 static int smp_cpu_state[NR_CPUS];
  72 static int cpu_management;
  73
  74 static DEFINE_PER_CPU(struct cpu, cpu_devices);
  75
  76 static void smp_ext_bitcall(int, int);
  77
  78 static int raw_cpu_stopped(int cpu)
  79 {
  80         u32 status;
  81
  82         switch (raw_sigp_ps(&status, 0, cpu, sigp_sense)) {
  83         case sigp_status_stored:
  84                 /* Check for stopped and check stop state */
  85                 if (status & 0x50)
  86                         return 1;
  87                 break;
  88         default:
  89                 break;
  90         }
  91         return 0;
  92 }
  93
  94 static inline int cpu_stopped(int cpu)
  95 {
  96         return raw_cpu_stopped(cpu_logical_map(cpu));
  97 }
  98
  99 void smp_switch_to_ipl_cpu(void (*func)(void *), void *data)
 100 {
 101         struct _lowcore *lc, *current_lc;
 102         struct stack_frame *sf;
 103         struct pt_regs *regs;
 104         unsigned long sp;
 105
 106         if (smp_processor_id() == 0)
 107                 func(data);
 108         __load_psw_mask(PSW_BASE_BITS | PSW_DEFAULT_KEY);
 109         /* Disable lowcore protection */
 110         __ctl_clear_bit(0, 28);
 111         current_lc = lowcore_ptr[smp_processor_id()];
 112         lc = lowcore_ptr[0];
 113         if (!lc)
 114                 lc = current_lc;
 115         lc->restart_psw.mask = PSW_BASE_BITS | PSW_DEFAULT_KEY;
 116         lc->restart_psw.addr = PSW_ADDR_AMODE | (unsigned long) smp_restart_cpu;
 117         if (!cpu_online(0))
 118                 smp_switch_to_cpu(func, data, 0, stap(), __cpu_logical_map[0]);
 119         while (sigp(0, sigp_stop_and_store_status) == sigp_busy)
 120                 cpu_relax();
 121         sp = lc->panic_stack;
 122         sp -= sizeof(struct pt_regs);
 123         regs = (struct pt_regs *) sp;
 124         memcpy(&regs->gprs, &current_lc->gpregs_save_area, sizeof(regs->gprs));
 125         regs->psw = lc->psw_save_area;
 126         sp -= STACK_FRAME_OVERHEAD;
 127         sf = (struct stack_frame *) sp;
 128         sf->back_chain = regs->gprs[15];
 129         smp_switch_to_cpu(func, data, sp, stap(), __cpu_logical_map[0]);
 130 }
 131
 132 void smp_send_stop(void)
 133 {
 134         int cpu, rc;
 135
 136         /* Disable all interrupts/machine checks */
 137         __load_psw_mask(psw_kernel_bits & ~PSW_MASK_MCHECK);
 138         trace_hardirqs_off();
 139
 140         /* stop all processors */
 141         for_each_online_cpu(cpu) {
 142                 if (cpu == smp_processor_id())
 143                         continue;
 144                 do {
 145                         rc = sigp(cpu, sigp_stop);
 146                 } while (rc == sigp_busy);
 147
 148                 while (!cpu_stopped(cpu))
 149                         cpu_relax();
 150         }
 151 }
 152
 153 /*
 154  * This is the main routine where commands issued by other
 155  * cpus are handled.
 156  */
 157
 158 static void do_ext_call_interrupt(__u16 code)
 159 {
 160         unsigned long bits;
 161
 162         /*
 163          * handle bit signal external calls
 164          *
 165          * For the ec_schedule signal we have to do nothing. All the work
 166          * is done automatically when we return from the interrupt.
 167          */
 168         bits = xchg(&S390_lowcore.ext_call_fast, 0);
 169
 170         if (test_bit(ec_call_function, &bits))
 171                 generic_smp_call_function_interrupt();
 172
 173         if (test_bit(ec_call_function_single, &bits))
 174                 generic_smp_call_function_single_interrupt();
 175 }
 176
 177 /*
 178  * Send an external call sigp to another cpu and return without waiting
 179  * for its completion.
 180  */
 181 static void smp_ext_bitcall(int cpu, int sig)
 182 {
 183         /*
 184          * Set signaling bit in lowcore of target cpu and kick it
 185          */
 186         set_bit(sig, (unsigned long *) &lowcore_ptr[cpu]->ext_call_fast);
 187         while (sigp(cpu, sigp_emergency_signal) == sigp_busy)
 188                 udelay(10);
 189 }
 190
 191 void arch_send_call_function_ipi_mask(const struct cpumask *mask)
 192 {
 193         int cpu;
 194
 195         for_each_cpu(cpu, mask)
 196                 smp_ext_bitcall(cpu, ec_call_function);
 197 }
 198
 199 void arch_send_call_function_single_ipi(int cpu)
 200 {
 201         smp_ext_bitcall(cpu, ec_call_function_single);
 202 }
 203
 204 #ifndef CONFIG_64BIT
 205 /*
 206  * this function sends a 'purge tlb' signal to another CPU.
 207  */
 208 static void smp_ptlb_callback(void *info)
 209 {
 210         __tlb_flush_local();
 211 }
 212
 213 void smp_ptlb_all(void)
 214 {
 215         on_each_cpu(smp_ptlb_callback, NULL, 1);
 216 }
 217 EXPORT_SYMBOL(smp_ptlb_all);
 218 #endif /* ! CONFIG_64BIT */
 219
 220 /*
 221  * this function sends a 'reschedule' IPI to another CPU.
 222  * it goes straight through and wastes no time serializing
 223  * anything. Worst case is that we lose a reschedule ...
 224  */
 225 void smp_send_reschedule(int cpu)
 226 {
 227         smp_ext_bitcall(cpu, ec_schedule);
 228 }
 229
 230 /*
 231  * parameter area for the set/clear control bit callbacks
 232  */
 233 struct ec_creg_mask_parms {
 234         unsigned long orvals[16];
 235         unsigned long andvals[16];
 236 };
 237
 238 /*
 239  * callback for setting/clearing control bits
 240  */
 241 static void smp_ctl_bit_callback(void *info)
 242 {
 243         struct ec_creg_mask_parms *pp = info;
 244         unsigned long cregs[16];
 245         int i;
 246
 247         __ctl_store(cregs, 0, 15);
 248         for (i = 0; i <= 15; i++)
 249                 cregs[i] = (cregs[i] & pp->andvals[i]) | pp->orvals[i];
 250         __ctl_load(cregs, 0, 15);
 251 }
 252
 253 /*
 254  * Set a bit in a control register of all cpus
 255  */
 256 void smp_ctl_set_bit(int cr, int bit)
 257 {
 258         struct ec_creg_mask_parms parms;
 259
 260         memset(&parms.orvals, 0, sizeof(parms.orvals));
 261         memset(&parms.andvals, 0xff, sizeof(parms.andvals));
 262         parms.orvals[cr] = 1 << bit;
 263         on_each_cpu(smp_ctl_bit_callback, &parms, 1);
 264 }
 265 EXPORT_SYMBOL(smp_ctl_set_bit);
 266
 267 /*
 268  * Clear a bit in a control register of all cpus
 269  */
 270 void smp_ctl_clear_bit(int cr, int bit)
 271 {
 272         struct ec_creg_mask_parms parms;
 273
 274         memset(&parms.orvals, 0, sizeof(parms.orvals));
 275         memset(&parms.andvals, 0xff, sizeof(parms.andvals));
 276         parms.andvals[cr] = ~(1L << bit);
 277         on_each_cpu(smp_ctl_bit_callback, &parms, 1);
 278 }
 279 EXPORT_SYMBOL(smp_ctl_clear_bit);
 280
 281 #ifdef CONFIG_ZFCPDUMP
 282
 283 static void __init smp_get_save_area(unsigned int cpu, unsigned int phy_cpu)
 284 {
 285         if (ipl_info.type != IPL_TYPE_FCP_DUMP)
 286                 return;
 287         if (cpu >= NR_CPUS) {
 288                 pr_warning("CPU %i exceeds the maximum %i and is excluded from "
 289                            "the dump\n", cpu, NR_CPUS - 1);
 290                 return;
 291         }
 292         zfcpdump_save_areas[cpu] = kmalloc(sizeof(struct save_area), GFP_KERNEL);
 293         while (raw_sigp(phy_cpu, sigp_stop_and_store_status) == sigp_busy)
 294                 cpu_relax();
 295         memcpy(zfcpdump_save_areas[cpu],
 296                (void *)(unsigned long) store_prefix() + SAVE_AREA_BASE,
 297                sizeof(struct save_area));
 298 }
 299
 300 struct save_area *zfcpdump_save_areas[NR_CPUS + 1];
 301 EXPORT_SYMBOL_GPL(zfcpdump_save_areas);
 302
 303 #else
 304
 305 static inline void smp_get_save_area(unsigned int cpu, unsigned int phy_cpu) { }
 306
 307 #endif /* CONFIG_ZFCPDUMP */
 308
 309 static int cpu_known(int cpu_id)
 310 {
 311         int cpu;
 312
 313         for_each_present_cpu(cpu) {
 314                 if (__cpu_logical_map[cpu] == cpu_id)
 315                         return 1;
 316         }
 317         return 0;
 318 }
 319
 320 static int smp_rescan_cpus_sigp(cpumask_t avail)
 321 {
 322         int cpu_id, logical_cpu;
 323
 324         logical_cpu = cpumask_first(&avail);
 325         if (logical_cpu >= nr_cpu_ids)
 326                 return 0;
 327         for (cpu_id = 0; cpu_id <= MAX_CPU_ADDRESS; cpu_id++) {
 328                 if (cpu_known(cpu_id))
 329                         continue;
 330                 __cpu_logical_map[logical_cpu] = cpu_id;
 331                 smp_cpu_polarization[logical_cpu] = POLARIZATION_UNKNWN;
 332                 if (!cpu_stopped(logical_cpu))
 333                         continue;
 334                 cpu_set(logical_cpu, cpu_present_map);
 335                 smp_cpu_state[logical_cpu] = CPU_STATE_CONFIGURED;
 336                 logical_cpu = cpumask_next(logical_cpu, &avail);
 337                 if (logical_cpu >= nr_cpu_ids)
 338                         break;
 339         }
 340         return 0;
 341 }
 342
 343 static int smp_rescan_cpus_sclp(cpumask_t avail)
 344 {
 345         struct sclp_cpu_info *info;
 346         int cpu_id, logical_cpu, cpu;
 347         int rc;
 348
 349         logical_cpu = cpumask_first(&avail);
 350         if (logical_cpu >= nr_cpu_ids)
 351                 return 0;
 352         info = kmalloc(sizeof(*info), GFP_KERNEL);
 353         if (!info)
 354                 return -ENOMEM;
 355         rc = sclp_get_cpu_info(info);
 356         if (rc)
 357                 goto out;
 358         for (cpu = 0; cpu < info->combined; cpu++) {
 359                 if (info->has_cpu_type && info->cpu[cpu].type != smp_cpu_type)
 360                         continue;
 361                 cpu_id = info->cpu[cpu].address;
 362                 if (cpu_known(cpu_id))
 363                         continue;
 364                 __cpu_logical_map[logical_cpu] = cpu_id;
 365                 smp_cpu_polarization[logical_cpu] = POLARIZATION_UNKNWN;
 366                 cpu_set(logical_cpu, cpu_present_map);
 367                 if (cpu >= info->configured)
 368                         smp_cpu_state[logical_cpu] = CPU_STATE_STANDBY;
 369                 else
 370                         smp_cpu_state[logical_cpu] = CPU_STATE_CONFIGURED;
 371                 logical_cpu = cpumask_next(logical_cpu, &avail);
 372                 if (logical_cpu >= nr_cpu_ids)
 373                         break;
 374         }
 375 out:
 376         kfree(info);
 377         return rc;
 378 }
 379
 380 static int __smp_rescan_cpus(void)
 381 {
 382         cpumask_t avail;
 383
 384         cpus_xor(avail, cpu_possible_map, cpu_present_map);
 385         if (smp_use_sigp_detection)
 386                 return smp_rescan_cpus_sigp(avail);
 387         else
 388                 return smp_rescan_cpus_sclp(avail);
 389 }
 390
 391 static void __init smp_detect_cpus(void)
 392 {
 393         unsigned int cpu, c_cpus, s_cpus;
 394         struct sclp_cpu_info *info;
 395         u16 boot_cpu_addr, cpu_addr;
 396
 397         c_cpus = 1;
 398         s_cpus = 0;
 399         boot_cpu_addr = __cpu_logical_map[0];
 400         info = kmalloc(sizeof(*info), GFP_KERNEL);
 401         if (!info)
 402                 panic("smp_detect_cpus failed to allocate memory\n");
 403         /* Use sigp detection algorithm if sclp doesn't work. */
 404         if (sclp_get_cpu_info(info)) {
 405                 smp_use_sigp_detection = 1;
 406                 for (cpu = 0; cpu <= MAX_CPU_ADDRESS; cpu++) {
 407                         if (cpu == boot_cpu_addr)
 408                                 continue;
 409                         if (!raw_cpu_stopped(cpu))
 410                                 continue;
 411                         smp_get_save_area(c_cpus, cpu);
 412                         c_cpus++;
 413                 }
 414                 goto out;
 415         }
 416
 417         if (info->has_cpu_type) {
 418                 for (cpu = 0; cpu < info->combined; cpu++) {
 419                         if (info->cpu[cpu].address == boot_cpu_addr) {
 420                                 smp_cpu_type = info->cpu[cpu].type;
 421                                 break;
 422                         }
 423                 }
 424         }
 425
 426         for (cpu = 0; cpu < info->combined; cpu++) {
 427                 if (info->has_cpu_type && info->cpu[cpu].type != smp_cpu_type)
 428                         continue;
 429                 cpu_addr = info->cpu[cpu].address;
 430                 if (cpu_addr == boot_cpu_addr)
 431                         continue;
 432                 if (!raw_cpu_stopped(cpu_addr)) {
 433                         s_cpus++;
 434                         continue;
 435                 }
 436                 smp_get_save_area(c_cpus, cpu_addr);
 437                 c_cpus++;
 438         }
 439 out:
 440         kfree(info);
 441         pr_info("%d configured CPUs, %d standby CPUs\n", c_cpus, s_cpus);
 442         get_online_cpus();
 443         __smp_rescan_cpus();
 444         put_online_cpus();
 445 }
 446
 447 /*
 448  *      Activate a secondary processor.
 449  */
 450 int __cpuinit start_secondary(void *cpuvoid)
 451 {
 452         /* Setup the cpu */
 453         cpu_init();
 454         preempt_disable();
 455         /* Enable TOD clock interrupts on the secondary cpu. */
 456         init_cpu_timer();
 457         /* Enable cpu timer interrupts on the secondary cpu. */
 458         init_cpu_vtimer();
 459         /* Enable pfault pseudo page faults on this cpu. */
 460         pfault_init();
 461
 462         /* call cpu notifiers */
 463         notify_cpu_starting(smp_processor_id());
 464         /* Mark this cpu as online */
 465         ipi_call_lock();
 466         cpu_set(smp_processor_id(), cpu_online_map);
 467         ipi_call_unlock();
 468         /* Switch on interrupts */
 469         local_irq_enable();
 470         /* Print info about this processor */
 471         print_cpu_info();
 472         /* cpu_idle will call schedule for us */
 473         cpu_idle();
 474         return 0;
 475 }
 476
 477 static void __init smp_create_idle(unsigned int cpu)
 478 {
 479         struct task_struct *p;
 480
 481         /*
 482          *  don't care about the psw and regs settings since we'll never
 483          *  reschedule the forked task.
 484          */
 485         p = fork_idle(cpu);
 486         if (IS_ERR(p))
 487                 panic("failed fork for CPU %u: %li", cpu, PTR_ERR(p));
 488         current_set[cpu] = p;
 489 }
 490
 491 static int __cpuinit smp_alloc_lowcore(int cpu)
 492 {
 493         unsigned long async_stack, panic_stack;
 494         struct _lowcore *lowcore;
 495
 496         lowcore = (void *) __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
 497         if (!lowcore)
 498                 return -ENOMEM;
 499         async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER);
 500         panic_stack = __get_free_page(GFP_KERNEL);
 501         if (!panic_stack || !async_stack)
 502                 goto out;
 503         memcpy(lowcore, &S390_lowcore, 512);
 504         memset((char *)lowcore + 512, 0, sizeof(*lowcore) - 512);
 505         lowcore->async_stack = async_stack + ASYNC_SIZE;
 506         lowcore->panic_stack = panic_stack + PAGE_SIZE;
 507
 508 #ifndef CONFIG_64BIT
 509         if (MACHINE_HAS_IEEE) {
 510                 unsigned long save_area;
 511
 512                 save_area = get_zeroed_page(GFP_KERNEL);
 513                 if (!save_area)
 514                         goto out;
 515                 lowcore->extended_save_area_addr = (u32) save_area;
 516         }
 517 #else
 518         if (vdso_alloc_per_cpu(cpu, lowcore))
 519                 goto out;
 520 #endif
 521         lowcore_ptr[cpu] = lowcore;
 522         return 0;
 523
 524 out:
 525         free_page(panic_stack);
 526         free_pages(async_stack, ASYNC_ORDER);
 527         free_pages((unsigned long) lowcore, LC_ORDER);
 528         return -ENOMEM;
 529 }
 530
 531 static void smp_free_lowcore(int cpu)
 532 {
 533         struct _lowcore *lowcore;
 534
 535         lowcore = lowcore_ptr[cpu];
 536 #ifndef CONFIG_64BIT
 537         if (MACHINE_HAS_IEEE)
 538                 free_page((unsigned long) lowcore->extended_save_area_addr);
 539 #else
 540         vdso_free_per_cpu(cpu, lowcore);
 541 #endif
 542         free_page(lowcore->panic_stack - PAGE_SIZE);
 543         free_pages(lowcore->async_stack - ASYNC_SIZE, ASYNC_ORDER);
 544         free_pages((unsigned long) lowcore, LC_ORDER);
 545         lowcore_ptr[cpu] = NULL;
 546 }
 547
 548 /* Upping and downing of CPUs */
 549 int __cpuinit __cpu_up(unsigned int cpu)
 550 {
 551         struct _lowcore *cpu_lowcore;
 552         struct task_struct *idle;
 553         struct stack_frame *sf;
 554         u32 lowcore;
 555         int ccode;
 556
 557         if (smp_cpu_state[cpu] != CPU_STATE_CONFIGURED)
 558                 return -EIO;
 559         if (smp_alloc_lowcore(cpu))
 560                 return -ENOMEM;
 561         do {
 562                 ccode = sigp(cpu, sigp_initial_cpu_reset);
 563                 if (ccode == sigp_busy)
 564                         udelay(10);
 565                 if (ccode == sigp_not_operational)
 566                         goto err_out;
 567         } while (ccode == sigp_busy);
 568
 569         lowcore = (u32)(unsigned long)lowcore_ptr[cpu];
 570         while (sigp_p(lowcore, cpu, sigp_set_prefix) == sigp_busy)
 571                 udelay(10);
 572
 573         idle = current_set[cpu];
 574         cpu_lowcore = lowcore_ptr[cpu];
 575         cpu_lowcore->kernel_stack = (unsigned long)
 576                 task_stack_page(idle) + THREAD_SIZE;
 577         cpu_lowcore->thread_info = (unsigned long) task_thread_info(idle);
 578         sf = (struct stack_frame *) (cpu_lowcore->kernel_stack
 579                                      - sizeof(struct pt_regs)
 580                                      - sizeof(struct stack_frame));
 581         memset(sf, 0, sizeof(struct stack_frame));
 582         sf->gprs[9] = (unsigned long) sf;
 583         cpu_lowcore->save_area[15] = (unsigned long) sf;
 584         __ctl_store(cpu_lowcore->cregs_save_area, 0, 15);
 585         asm volatile(
 586                 "       stam    0,15,0(%0)"
 587                 : : "a" (&cpu_lowcore->access_regs_save_area) : "memory");
 588         cpu_lowcore->percpu_offset = __per_cpu_offset[cpu];
 589         cpu_lowcore->current_task = (unsigned long) idle;
 590         cpu_lowcore->cpu_nr = cpu;
 591         cpu_lowcore->kernel_asce = S390_lowcore.kernel_asce;
 592         cpu_lowcore->machine_flags = S390_lowcore.machine_flags;
 593         cpu_lowcore->ftrace_func = S390_lowcore.ftrace_func;
 594         eieio();
 595
 596         while (sigp(cpu, sigp_restart) == sigp_busy)
 597                 udelay(10);
 598
 599         while (!cpu_online(cpu))
 600                 cpu_relax();
 601         return 0;
 602
 603 err_out:
 604         smp_free_lowcore(cpu);
 605         return -EIO;
 606 }
 607
 608 static int __init setup_possible_cpus(char *s)
 609 {
 610         int pcpus, cpu;
 611
 612         pcpus = simple_strtoul(s, NULL, 0);
 613         init_cpu_possible(cpumask_of(0));
 614         for (cpu = 1; cpu < pcpus && cpu < nr_cpu_ids; cpu++)
 615                 set_cpu_possible(cpu, true);
 616         return 0;
 617 }
 618 early_param("possible_cpus", setup_possible_cpus);
 619
 620 #ifdef CONFIG_HOTPLUG_CPU
 621
 622 int __cpu_disable(void)
 623 {
 624         struct ec_creg_mask_parms cr_parms;
 625         int cpu = smp_processor_id();
 626
 627         cpu_clear(cpu, cpu_online_map);
 628
 629         /* Disable pfault pseudo page faults on this cpu. */
 630         pfault_fini();
 631
 632         memset(&cr_parms.orvals, 0, sizeof(cr_parms.orvals));
 633         memset(&cr_parms.andvals, 0xff, sizeof(cr_parms.andvals));
 634
 635         /* disable all external interrupts */
 636         cr_parms.orvals[0] = 0;
 637         cr_parms.andvals[0] = ~(1 << 15 | 1 << 14 | 1 << 13 | 1 << 12 |
 638                                 1 << 11 | 1 << 10 | 1 <<  6 | 1 <<  4);
 639         /* disable all I/O interrupts */
 640         cr_parms.orvals[6] = 0;
 641         cr_parms.andvals[6] = ~(1 << 31 | 1 << 30 | 1 << 29 | 1 << 28 |
 642                                 1 << 27 | 1 << 26 | 1 << 25 | 1 << 24);
 643         /* disable most machine checks */
 644         cr_parms.orvals[14] = 0;
 645         cr_parms.andvals[14] = ~(1 << 28 | 1 << 27 | 1 << 26 |
 646                                  1 << 25 | 1 << 24);
 647
 648         smp_ctl_bit_callback(&cr_parms);
 649
 650         return 0;
 651 }
 652
 653 void __cpu_die(unsigned int cpu)
 654 {
 655         /* Wait until target cpu is down */
 656         while (!cpu_stopped(cpu))
 657                 cpu_relax();
 658         while (sigp_p(0, cpu, sigp_set_prefix) == sigp_busy)
 659                 udelay(10);
 660         smp_free_lowcore(cpu);
 661         pr_info("Processor %d stopped\n", cpu);
 662 }
 663
 664 void cpu_die(void)
 665 {
 666         idle_task_exit();
 667         while (sigp(smp_processor_id(), sigp_stop) == sigp_busy)
 668                 cpu_relax();
 669         for (;;);
 670 }
 671
 672 #endif /* CONFIG_HOTPLUG_CPU */
 673
 674 void __init smp_prepare_cpus(unsigned int max_cpus)
 675 {
 676 #ifndef CONFIG_64BIT
 677         unsigned long save_area = 0;
 678 #endif
 679         unsigned long async_stack, panic_stack;
 680         struct _lowcore *lowcore;
 681         unsigned int cpu;
 682
 683         smp_detect_cpus();
 684
 685         /* request the 0x1201 emergency signal external interrupt */
 686         if (register_external_interrupt(0x1201, do_ext_call_interrupt) != 0)
 687                 panic("Couldn't request external interrupt 0x1201");
 688         print_cpu_info();
 689
 690         /* Reallocate current lowcore, but keep its contents. */
 691         lowcore = (void *) __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
 692         panic_stack = __get_free_page(GFP_KERNEL);
 693         async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER);
 694         BUG_ON(!lowcore || !panic_stack || !async_stack);
 695 #ifndef CONFIG_64BIT
 696         if (MACHINE_HAS_IEEE)
 697                 save_area = get_zeroed_page(GFP_KERNEL);
 698 #endif
 699         local_irq_disable();
 700         local_mcck_disable();
 701         lowcore_ptr[smp_processor_id()] = lowcore;
 702         *lowcore = S390_lowcore;
 703         lowcore->panic_stack = panic_stack + PAGE_SIZE;
 704         lowcore->async_stack = async_stack + ASYNC_SIZE;
 705 #ifndef CONFIG_64BIT
 706         if (MACHINE_HAS_IEEE)
 707                 lowcore->extended_save_area_addr = (u32) save_area;
 708 #endif
 709         set_prefix((u32)(unsigned long) lowcore);
 710         local_mcck_enable();
 711         local_irq_enable();
 712 #ifdef CONFIG_64BIT
 713         if (vdso_alloc_per_cpu(smp_processor_id(), &S390_lowcore))
 714                 BUG();
 715 #endif
 716         for_each_possible_cpu(cpu)
 717                 if (cpu != smp_processor_id())
 718                         smp_create_idle(cpu);
 719 }
 720
 721 void __init smp_prepare_boot_cpu(void)
 722 {
 723         BUG_ON(smp_processor_id() != 0);
 724
 725         current_thread_info()->cpu = 0;
 726         cpu_set(0, cpu_present_map);
 727         cpu_set(0, cpu_online_map);
 728         S390_lowcore.percpu_offset = __per_cpu_offset[0];
 729         current_set[0] = current;
 730         smp_cpu_state[0] = CPU_STATE_CONFIGURED;
 731         smp_cpu_polarization[0] = POLARIZATION_UNKNWN;
 732 }
 733
 734 void __init smp_cpus_done(unsigned int max_cpus)
 735 {
 736 }
 737
 738 void __init smp_setup_processor_id(void)
 739 {
 740         S390_lowcore.cpu_nr = 0;
 741         __cpu_logical_map[0] = stap();
 742 }
 743
 744 /*
 745  * the frequency of the profiling timer can be changed
 746  * by writing a multiplier value into /proc/profile.
 747  *
 748  * usually you want to run this on all CPUs ;)
 749  */
 750 int setup_profiling_timer(unsigned int multiplier)
 751 {
 752         return 0;
 753 }
 754
 755 #ifdef CONFIG_HOTPLUG_CPU
 756 static ssize_t cpu_configure_show(struct sys_device *dev,
 757                                 struct sysdev_attribute *attr, char *buf)
 758 {
 759         ssize_t count;
 760
 761         mutex_lock(&smp_cpu_state_mutex);
 762         count = sprintf(buf, "%d\n", smp_cpu_state[dev->id]);
 763         mutex_unlock(&smp_cpu_state_mutex);
 764         return count;
 765 }
 766
 767 static ssize_t cpu_configure_store(struct sys_device *dev,
 768                                   struct sysdev_attribute *attr,
 769                                   const char *buf, size_t count)
 770 {
 771         int cpu = dev->id;
 772         int val, rc;
 773         char delim;
 774
 775         if (sscanf(buf, "%d %c", &val, &delim) != 1)
 776                 return -EINVAL;
 777         if (val != 0 && val != 1)
 778                 return -EINVAL;
 779
 780         get_online_cpus();
 781         mutex_lock(&smp_cpu_state_mutex);
 782         rc = -EBUSY;
 783         /* disallow configuration changes of online cpus and cpu 0 */
 784         if (cpu_online(cpu) || cpu == 0)
 785                 goto out;
 786         rc = 0;
 787         switch (val) {
 788         case 0:
 789                 if (smp_cpu_state[cpu] == CPU_STATE_CONFIGURED) {
 790                         rc = sclp_cpu_deconfigure(__cpu_logical_map[cpu]);
 791                         if (!rc) {
 792                                 smp_cpu_state[cpu] = CPU_STATE_STANDBY;
 793                                 smp_cpu_polarization[cpu] = POLARIZATION_UNKNWN;
 794                         }
 795                 }
 796                 break;
 797         case 1:
 798                 if (smp_cpu_state[cpu] == CPU_STATE_STANDBY) {
 799                         rc = sclp_cpu_configure(__cpu_logical_map[cpu]);
 800                         if (!rc) {
 801                                 smp_cpu_state[cpu] = CPU_STATE_CONFIGURED;
 802                                 smp_cpu_polarization[cpu] = POLARIZATION_UNKNWN;
 803                         }
 804                 }
 805                 break;
 806         default:
 807                 break;
 808         }
 809 out:
 810         mutex_unlock(&smp_cpu_state_mutex);
 811         put_online_cpus();
 812         return rc ? rc : count;
 813 }
 814 static SYSDEV_ATTR(configure, 0644, cpu_configure_show, cpu_configure_store);
 815 #endif /* CONFIG_HOTPLUG_CPU */
 816
 817 static ssize_t cpu_polarization_show(struct sys_device *dev,
 818                                      struct sysdev_attribute *attr, char *buf)
 819 {
 820         int cpu = dev->id;
 821         ssize_t count;
 822
 823         mutex_lock(&smp_cpu_state_mutex);
 824         switch (smp_cpu_polarization[cpu]) {
 825         case POLARIZATION_HRZ:
 826                 count = sprintf(buf, "horizontal\n");
 827                 break;
 828         case POLARIZATION_VL:
 829                 count = sprintf(buf, "vertical:low\n");
 830                 break;
 831         case POLARIZATION_VM:
 832                 count = sprintf(buf, "vertical:medium\n");
 833                 break;
 834         case POLARIZATION_VH:
 835                 count = sprintf(buf, "vertical:high\n");
 836                 break;
 837         default:
 838                 count = sprintf(buf, "unknown\n");
 839                 break;
 840         }
 841         mutex_unlock(&smp_cpu_state_mutex);
 842         return count;
 843 }
 844 static SYSDEV_ATTR(polarization, 0444, cpu_polarization_show, NULL);
 845
 846 static ssize_t show_cpu_address(struct sys_device *dev,
 847                                 struct sysdev_attribute *attr, char *buf)
 848 {
 849         return sprintf(buf, "%d\n", __cpu_logical_map[dev->id]);
 850 }
 851 static SYSDEV_ATTR(address, 0444, show_cpu_address, NULL);
 852
 853
 854 static struct attribute *cpu_common_attrs[] = {
 855 #ifdef CONFIG_HOTPLUG_CPU
 856         &attr_configure.attr,
 857 #endif
 858         &attr_address.attr,
 859         &attr_polarization.attr,
 860         NULL,
 861 };
 862
 863 static struct attribute_group cpu_common_attr_group = {
 864         .attrs = cpu_common_attrs,
 865 };
 866
 867 static ssize_t show_capability(struct sys_device *dev,
 868                                 struct sysdev_attribute *attr, char *buf)
 869 {
 870         unsigned int capability;
 871         int rc;
 872
 873         rc = get_cpu_capability(&capability);
 874         if (rc)
 875                 return rc;
 876         return sprintf(buf, "%u\n", capability);
 877 }
 878 static SYSDEV_ATTR(capability, 0444, show_capability, NULL);
 879
 880 static ssize_t show_idle_count(struct sys_device *dev,
 881                                 struct sysdev_attribute *attr, char *buf)
 882 {
 883         struct s390_idle_data *idle;
 884         unsigned long long idle_count;
 885         unsigned int sequence;
 886
 887         idle = &per_cpu(s390_idle, dev->id);
 888 repeat:
 889         sequence = idle->sequence;
 890         smp_rmb();
 891         if (sequence & 1)
 892                 goto repeat;
 893         idle_count = idle->idle_count;
 894         if (idle->idle_enter)
 895                 idle_count++;
 896         smp_rmb();
 897         if (idle->sequence != sequence)
 898                 goto repeat;
 899         return sprintf(buf, "%llu\n", idle_count);
 900 }
 901 static SYSDEV_ATTR(idle_count, 0444, show_idle_count, NULL);
 902
 903 static ssize_t show_idle_time(struct sys_device *dev,
 904                                 struct sysdev_attribute *attr, char *buf)
 905 {
 906         struct s390_idle_data *idle;
 907         unsigned long long now, idle_time, idle_enter;
 908         unsigned int sequence;
 909
 910         idle = &per_cpu(s390_idle, dev->id);
 911         now = get_clock();
 912 repeat:
 913         sequence = idle->sequence;
 914         smp_rmb();
 915         if (sequence & 1)
 916                 goto repeat;
 917         idle_time = idle->idle_time;
 918         idle_enter = idle->idle_enter;
 919         if (idle_enter != 0ULL && idle_enter < now)
 920                 idle_time += now - idle_enter;
 921         smp_rmb();
 922         if (idle->sequence != sequence)
 923                 goto repeat;
 924         return sprintf(buf, "%llu\n", idle_time >> 12);
 925 }
 926 static SYSDEV_ATTR(idle_time_us, 0444, show_idle_time, NULL);
 927
 928 static struct attribute *cpu_online_attrs[] = {
 929         &attr_capability.attr,
 930         &attr_idle_count.attr,
 931         &attr_idle_time_us.attr,
 932         NULL,
 933 };
 934
 935 static struct attribute_group cpu_online_attr_group = {
 936         .attrs = cpu_online_attrs,
 937 };
 938
 939 static int __cpuinit smp_cpu_notify(struct notifier_block *self,
 940                                     unsigned long action, void *hcpu)
 941 {
 942         unsigned int cpu = (unsigned int)(long)hcpu;
 943         struct cpu *c = &per_cpu(cpu_devices, cpu);
 944         struct sys_device *s = &c->sysdev;
 945         struct s390_idle_data *idle;
 946
 947         switch (action) {
 948         case CPU_ONLINE:
 949         case CPU_ONLINE_FROZEN:
 950                 idle = &per_cpu(s390_idle, cpu);
 951                 memset(idle, 0, sizeof(struct s390_idle_data));
 952                 if (sysfs_create_group(&s->kobj, &cpu_online_attr_group))
 953                         return NOTIFY_BAD;
 954                 break;
 955         case CPU_DEAD:
 956         case CPU_DEAD_FROZEN:
 957                 sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
 958                 break;
 959         }
 960         return NOTIFY_OK;
 961 }
 962
 963 static struct notifier_block __cpuinitdata smp_cpu_nb = {
 964         .notifier_call = smp_cpu_notify,
 965 };
 966
 967 static int __devinit smp_add_present_cpu(int cpu)
 968 {
 969         struct cpu *c = &per_cpu(cpu_devices, cpu);
 970         struct sys_device *s = &c->sysdev;
 971         int rc;
 972
 973         c->hotpluggable = 1;
 974         rc = register_cpu(c, cpu);
 975         if (rc)
 976                 goto out;
 977         rc = sysfs_create_group(&s->kobj, &cpu_common_attr_group);
 978         if (rc)
 979                 goto out_cpu;
 980         if (!cpu_online(cpu))
 981                 goto out;
 982         rc = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
 983         if (!rc)
 984                 return 0;
 985         sysfs_remove_group(&s->kobj, &cpu_common_attr_group);
 986 out_cpu:
 987 #ifdef CONFIG_HOTPLUG_CPU
 988         unregister_cpu(c);
 989 #endif
 990 out:
 991         return rc;
 992 }
 993
 994 #ifdef CONFIG_HOTPLUG_CPU
 995
 996 int __ref smp_rescan_cpus(void)
 997 {
 998         cpumask_t newcpus;
 999         int cpu;
1000         int rc;
1001
1002         get_online_cpus();
1003         mutex_lock(&smp_cpu_state_mutex);
1004         newcpus = cpu_present_map;
1005         rc = __smp_rescan_cpus();
1006         if (rc)
1007                 goto out;
1008         cpus_andnot(newcpus, cpu_present_map, newcpus);
1009         for_each_cpu_mask(cpu, newcpus) {
1010                 rc = smp_add_present_cpu(cpu);
1011                 if (rc)
1012                         cpu_clear(cpu, cpu_present_map);
1013         }
1014         rc = 0;
1015 out:
1016         mutex_unlock(&smp_cpu_state_mutex);
1017         put_online_cpus();
1018         if (!cpus_empty(newcpus))
1019                 topology_schedule_update();
1020         return rc;
1021 }
1022
1023 static ssize_t __ref rescan_store(struct sysdev_class *class,
1024                                   struct sysdev_class_attribute *attr,
1025                                   const char *buf,
1026                                   size_t count)
1027 {
1028         int rc;
1029
1030         rc = smp_rescan_cpus();
1031         return rc ? rc : count;
1032 }
1033 static SYSDEV_CLASS_ATTR(rescan, 0200, NULL, rescan_store);
1034 #endif /* CONFIG_HOTPLUG_CPU */
1035
1036 static ssize_t dispatching_show(struct sysdev_class *class,
1037                                 struct sysdev_class_attribute *attr,
1038                                 char *buf)
1039 {
1040         ssize_t count;
1041
1042         mutex_lock(&smp_cpu_state_mutex);
1043         count = sprintf(buf, "%d\n", cpu_management);
1044         mutex_unlock(&smp_cpu_state_mutex);
1045         return count;
1046 }
1047
1048 static ssize_t dispatching_store(struct sysdev_class *dev,
1049                                  struct sysdev_class_attribute *attr,
1050                                  const char *buf,
1051                                  size_t count)
1052 {
1053         int val, rc;
1054         char delim;
1055
1056         if (sscanf(buf, "%d %c", &val, &delim) != 1)
1057                 return -EINVAL;
1058         if (val != 0 && val != 1)
1059                 return -EINVAL;
1060         rc = 0;
1061         get_online_cpus();
1062         mutex_lock(&smp_cpu_state_mutex);
1063         if (cpu_management == val)
1064                 goto out;
1065         rc = topology_set_cpu_management(val);
1066         if (!rc)
1067                 cpu_management = val;
1068 out:
1069         mutex_unlock(&smp_cpu_state_mutex);
1070         put_online_cpus();
1071         return rc ? rc : count;
1072 }
1073 static SYSDEV_CLASS_ATTR(dispatching, 0644, dispatching_show,
1074                          dispatching_store);
1075
1076 static int __init topology_init(void)
1077 {
1078         int cpu;
1079         int rc;
1080
1081         register_cpu_notifier(&smp_cpu_nb);
1082
1083 #ifdef CONFIG_HOTPLUG_CPU
1084         rc = sysdev_class_create_file(&cpu_sysdev_class, &attr_rescan);
1085         if (rc)
1086                 return rc;
1087 #endif
1088         rc = sysdev_class_create_file(&cpu_sysdev_class, &attr_dispatching);
1089         if (rc)
1090                 return rc;
1091         for_each_present_cpu(cpu) {
1092                 rc = smp_add_present_cpu(cpu);
1093                 if (rc)
1094                         return rc;
1095         }
1096         return 0;
1097 }
1098 subsys_initcall(topology_init);