kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  *      Distribute under GPLv2.
   7  *
   8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9  *
  10  *      Remote softirq infrastructure is by Jens Axboe.
  11  */
  12
  13 #include <linux/module.h>
  14 #include <linux/kernel_stat.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/init.h>
  17 #include <linux/mm.h>
  18 #include <linux/notifier.h>
  19 #include <linux/percpu.h>
  20 #include <linux/cpu.h>
  21 #include <linux/freezer.h>
  22 #include <linux/kthread.h>
  23 #include <linux/rcupdate.h>
  24 #include <linux/smp.h>
  25 #include <linux/tick.h>
  26
  27 #include <asm/irq.h>
  28 /*
  29    - No shared variables, all the data are CPU local.
  30    - If a softirq needs serialization, let it serialize itself
  31      by its own spinlocks.
  32    - Even if softirq is serialized, only local cpu is marked for
  33      execution. Hence, we get something sort of weak cpu binding.
  34      Though it is still not clear, will it result in better locality
  35      or will not.
  36
  37    Examples:
  38    - NET RX softirq. It is multithreaded and does not require
  39      any global serialization.
  40    - NET TX softirq. It kicks software netdevice queues, hence
  41      it is logically serialized per device, but this serialization
  42      is invisible to common code.
  43    - Tasklets: serialized wrt itself.
  44  */
  45
  46 #ifndef __ARCH_IRQ_STAT
  47 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  48 EXPORT_SYMBOL(irq_stat);
  49 #endif
  50
  51 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  52
  53 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  54
  55 /*
  56  * we cannot loop indefinitely here to avoid userspace starvation,
  57  * but we also don't want to introduce a worst case 1/HZ latency
  58  * to the pending events, so lets the scheduler to balance
  59  * the softirq load for us.
  60  */
  61 static inline void wakeup_softirqd(void)
  62 {
  63         /* Interrupts are disabled: no need to stop preemption */
  64         struct task_struct *tsk = __get_cpu_var(ksoftirqd);
  65
  66         if (tsk && tsk->state != TASK_RUNNING)
  67                 wake_up_process(tsk);
  68 }
  69
  70 /*
  71  * This one is for softirq.c-internal use,
  72  * where hardirqs are disabled legitimately:
  73  */
  74 #ifdef CONFIG_TRACE_IRQFLAGS
  75 static void __local_bh_disable(unsigned long ip)
  76 {
  77         unsigned long flags;
  78
  79         WARN_ON_ONCE(in_irq());
  80
  81         raw_local_irq_save(flags);
  82         add_preempt_count(SOFTIRQ_OFFSET);
  83         /*
  84          * Were softirqs turned off above:
  85          */
  86         if (softirq_count() == SOFTIRQ_OFFSET)
  87                 trace_softirqs_off(ip);
  88         raw_local_irq_restore(flags);
  89 }
  90 #else /* !CONFIG_TRACE_IRQFLAGS */
  91 static inline void __local_bh_disable(unsigned long ip)
  92 {
  93         add_preempt_count(SOFTIRQ_OFFSET);
  94         barrier();
  95 }
  96 #endif /* CONFIG_TRACE_IRQFLAGS */
  97
  98 void local_bh_disable(void)
  99 {
 100         __local_bh_disable((unsigned long)__builtin_return_address(0));
 101 }
 102
 103 EXPORT_SYMBOL(local_bh_disable);
 104
 105 /*
 106  * Special-case - softirqs can safely be enabled in
 107  * cond_resched_softirq(), or by __do_softirq(),
 108  * without processing still-pending softirqs:
 109  */
 110 void _local_bh_enable(void)
 111 {
 112         WARN_ON_ONCE(in_irq());
 113         WARN_ON_ONCE(!irqs_disabled());
 114
 115         if (softirq_count() == SOFTIRQ_OFFSET)
 116                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 117         sub_preempt_count(SOFTIRQ_OFFSET);
 118 }
 119
 120 EXPORT_SYMBOL(_local_bh_enable);
 121
 122 static inline void _local_bh_enable_ip(unsigned long ip)
 123 {
 124         WARN_ON_ONCE(in_irq() || irqs_disabled());
 125 #ifdef CONFIG_TRACE_IRQFLAGS
 126         local_irq_disable();
 127 #endif
 128         /*
 129          * Are softirqs going to be turned on now:
 130          */
 131         if (softirq_count() == SOFTIRQ_OFFSET)
 132                 trace_softirqs_on(ip);
 133         /*
 134          * Keep preemption disabled until we are done with
 135          * softirq processing:
 136          */
 137         sub_preempt_count(SOFTIRQ_OFFSET - 1);
 138
 139         if (unlikely(!in_interrupt() && local_softirq_pending()))
 140                 do_softirq();
 141
 142         dec_preempt_count();
 143 #ifdef CONFIG_TRACE_IRQFLAGS
 144         local_irq_enable();
 145 #endif
 146         preempt_check_resched();
 147 }
 148
 149 void local_bh_enable(void)
 150 {
 151         _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 152 }
 153 EXPORT_SYMBOL(local_bh_enable);
 154
 155 void local_bh_enable_ip(unsigned long ip)
 156 {
 157         _local_bh_enable_ip(ip);
 158 }
 159 EXPORT_SYMBOL(local_bh_enable_ip);
 160
 161 /*
 162  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
 163  * and we fall back to softirqd after that.
 164  *
 165  * This number has been established via experimentation.
 166  * The two things to balance is latency against fairness -
 167  * we want to handle softirqs as soon as possible, but they
 168  * should not be able to lock up the box.
 169  */
 170 #define MAX_SOFTIRQ_RESTART 10
 171
 172 asmlinkage void __do_softirq(void)
 173 {
 174         struct softirq_action *h;
 175         __u32 pending;
 176         int max_restart = MAX_SOFTIRQ_RESTART;
 177         int cpu;
 178
 179         pending = local_softirq_pending();
 180         account_system_vtime(current);
 181
 182         __local_bh_disable((unsigned long)__builtin_return_address(0));
 183         trace_softirq_enter();
 184
 185         cpu = smp_processor_id();
 186 restart:
 187         /* Reset the pending bitmask before enabling irqs */
 188         set_softirq_pending(0);
 189
 190         local_irq_enable();
 191
 192         h = softirq_vec;
 193
 194         do {
 195                 if (pending & 1) {
 196                         int prev_count = preempt_count();
 197
 198                         h->action(h);
 199
 200                         if (unlikely(prev_count != preempt_count())) {
 201                                 printk(KERN_ERR "huh, entered softirq %td %p"
 202                                        "with preempt_count %08x,"
 203                                        " exited with %08x?\n", h - softirq_vec,
 204                                        h->action, prev_count, preempt_count());
 205                                 preempt_count() = prev_count;
 206                         }
 207
 208                         rcu_bh_qsctr_inc(cpu);
 209                 }
 210                 h++;
 211                 pending >>= 1;
 212         } while (pending);
 213
 214         local_irq_disable();
 215
 216         pending = local_softirq_pending();
 217         if (pending && --max_restart)
 218                 goto restart;
 219
 220         if (pending)
 221                 wakeup_softirqd();
 222
 223         trace_softirq_exit();
 224
 225         account_system_vtime(current);
 226         _local_bh_enable();
 227 }
 228
 229 #ifndef __ARCH_HAS_DO_SOFTIRQ
 230
 231 asmlinkage void do_softirq(void)
 232 {
 233         __u32 pending;
 234         unsigned long flags;
 235
 236         if (in_interrupt())
 237                 return;
 238
 239         local_irq_save(flags);
 240
 241         pending = local_softirq_pending();
 242
 243         if (pending)
 244                 __do_softirq();
 245
 246         local_irq_restore(flags);
 247 }
 248
 249 #endif
 250
 251 /*
 252  * Enter an interrupt context.
 253  */
 254 void irq_enter(void)
 255 {
 256         int cpu = smp_processor_id();
 257
 258         rcu_irq_enter();
 259         if (idle_cpu(cpu) && !in_interrupt()) {
 260                 __irq_enter();
 261                 tick_check_idle(cpu);
 262         } else
 263                 __irq_enter();
 264 }
 265
 266 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
 267 # define invoke_softirq()       __do_softirq()
 268 #else
 269 # define invoke_softirq()       do_softirq()
 270 #endif
 271
 272 /*
 273  * Exit an interrupt context. Process softirqs if needed and possible:
 274  */
 275 void irq_exit(void)
 276 {
 277         account_system_vtime(current);
 278         trace_hardirq_exit();
 279         sub_preempt_count(IRQ_EXIT_OFFSET);
 280         if (!in_interrupt() && local_softirq_pending())
 281                 invoke_softirq();
 282
 283 #ifdef CONFIG_NO_HZ
 284         /* Make sure that timer wheel updates are propagated */
 285         rcu_irq_exit();
 286         if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
 287                 tick_nohz_stop_sched_tick(0);
 288 #endif
 289         preempt_enable_no_resched();
 290 }
 291
 292 /*
 293  * This function must run with irqs disabled!
 294  */
 295 inline void raise_softirq_irqoff(unsigned int nr)
 296 {
 297         __raise_softirq_irqoff(nr);
 298
 299         /*
 300          * If we're in an interrupt or softirq, we're done
 301          * (this also catches softirq-disabled code). We will
 302          * actually run the softirq once we return from
 303          * the irq or softirq.
 304          *
 305          * Otherwise we wake up ksoftirqd to make sure we
 306          * schedule the softirq soon.
 307          */
 308         if (!in_interrupt())
 309                 wakeup_softirqd();
 310 }
 311
 312 void raise_softirq(unsigned int nr)
 313 {
 314         unsigned long flags;
 315
 316         local_irq_save(flags);
 317         raise_softirq_irqoff(nr);
 318         local_irq_restore(flags);
 319 }
 320
 321 void open_softirq(int nr, void (*action)(struct softirq_action *))
 322 {
 323         softirq_vec[nr].action = action;
 324 }
 325
 326 /* Tasklets */
 327 struct tasklet_head
 328 {
 329         struct tasklet_struct *head;
 330         struct tasklet_struct **tail;
 331 };
 332
 333 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 334 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 335
 336 void __tasklet_schedule(struct tasklet_struct *t)
 337 {
 338         unsigned long flags;
 339
 340         local_irq_save(flags);
 341         t->next = NULL;
 342         *__get_cpu_var(tasklet_vec).tail = t;
 343         __get_cpu_var(tasklet_vec).tail = &(t->next);
 344         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 345         local_irq_restore(flags);
 346 }
 347
 348 EXPORT_SYMBOL(__tasklet_schedule);
 349
 350 void __tasklet_hi_schedule(struct tasklet_struct *t)
 351 {
 352         unsigned long flags;
 353
 354         local_irq_save(flags);
 355         t->next = NULL;
 356         *__get_cpu_var(tasklet_hi_vec).tail = t;
 357         __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
 358         raise_softirq_irqoff(HI_SOFTIRQ);
 359         local_irq_restore(flags);
 360 }
 361
 362 EXPORT_SYMBOL(__tasklet_hi_schedule);
 363
 364 static void tasklet_action(struct softirq_action *a)
 365 {
 366         struct tasklet_struct *list;
 367
 368         local_irq_disable();
 369         list = __get_cpu_var(tasklet_vec).head;
 370         __get_cpu_var(tasklet_vec).head = NULL;
 371         __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
 372         local_irq_enable();
 373
 374         while (list) {
 375                 struct tasklet_struct *t = list;
 376
 377                 list = list->next;
 378
 379                 if (tasklet_trylock(t)) {
 380                         if (!atomic_read(&t->count)) {
 381                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 382                                         BUG();
 383                                 t->func(t->data);
 384                                 tasklet_unlock(t);
 385                                 continue;
 386                         }
 387                         tasklet_unlock(t);
 388                 }
 389
 390                 local_irq_disable();
 391                 t->next = NULL;
 392                 *__get_cpu_var(tasklet_vec).tail = t;
 393                 __get_cpu_var(tasklet_vec).tail = &(t->next);
 394                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 395                 local_irq_enable();
 396         }
 397 }
 398
 399 static void tasklet_hi_action(struct softirq_action *a)
 400 {
 401         struct tasklet_struct *list;
 402
 403         local_irq_disable();
 404         list = __get_cpu_var(tasklet_hi_vec).head;
 405         __get_cpu_var(tasklet_hi_vec).head = NULL;
 406         __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
 407         local_irq_enable();
 408
 409         while (list) {
 410                 struct tasklet_struct *t = list;
 411
 412                 list = list->next;
 413
 414                 if (tasklet_trylock(t)) {
 415                         if (!atomic_read(&t->count)) {
 416                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 417                                         BUG();
 418                                 t->func(t->data);
 419                                 tasklet_unlock(t);
 420                                 continue;
 421                         }
 422                         tasklet_unlock(t);
 423                 }
 424
 425                 local_irq_disable();
 426                 t->next = NULL;
 427                 *__get_cpu_var(tasklet_hi_vec).tail = t;
 428                 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
 429                 __raise_softirq_irqoff(HI_SOFTIRQ);
 430                 local_irq_enable();
 431         }
 432 }
 433
 434
 435 void tasklet_init(struct tasklet_struct *t,
 436                   void (*func)(unsigned long), unsigned long data)
 437 {
 438         t->next = NULL;
 439         t->state = 0;
 440         atomic_set(&t->count, 0);
 441         t->func = func;
 442         t->data = data;
 443 }
 444
 445 EXPORT_SYMBOL(tasklet_init);
 446
 447 void tasklet_kill(struct tasklet_struct *t)
 448 {
 449         if (in_interrupt())
 450                 printk("Attempt to kill tasklet from interrupt\n");
 451
 452         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 453                 do
 454                         yield();
 455                 while (test_bit(TASKLET_STATE_SCHED, &t->state));
 456         }
 457         tasklet_unlock_wait(t);
 458         clear_bit(TASKLET_STATE_SCHED, &t->state);
 459 }
 460
 461 EXPORT_SYMBOL(tasklet_kill);
 462
 463 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
 464 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
 465
 466 static void __local_trigger(struct call_single_data *cp, int softirq)
 467 {
 468         struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
 469
 470         list_add_tail(&cp->list, head);
 471
 472         /* Trigger the softirq only if the list was previously empty.  */
 473         if (head->next == &cp->list)
 474                 raise_softirq_irqoff(softirq);
 475 }
 476
 477 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
 478 static void remote_softirq_receive(void *data)
 479 {
 480         struct call_single_data *cp = data;
 481         unsigned long flags;
 482         int softirq;
 483
 484         softirq = cp->priv;
 485
 486         local_irq_save(flags);
 487         __local_trigger(cp, softirq);
 488         local_irq_restore(flags);
 489 }
 490
 491 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 492 {
 493         if (cpu_online(cpu)) {
 494                 cp->func = remote_softirq_receive;
 495                 cp->info = cp;
 496                 cp->flags = 0;
 497                 cp->priv = softirq;
 498
 499                 __smp_call_function_single(cpu, cp);
 500                 return 0;
 501         }
 502         return 1;
 503 }
 504 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
 505 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 506 {
 507         return 1;
 508 }
 509 #endif
 510
 511 /**
 512  * __send_remote_softirq - try to schedule softirq work on a remote cpu
 513  * @cp: private SMP call function data area
 514  * @cpu: the remote cpu
 515  * @this_cpu: the currently executing cpu
 516  * @softirq: the softirq for the work
 517  *
 518  * Attempt to schedule softirq work on a remote cpu.  If this cannot be
 519  * done, the work is instead queued up on the local cpu.
 520  *
 521  * Interrupts must be disabled.
 522  */
 523 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
 524 {
 525         if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
 526                 __local_trigger(cp, softirq);
 527 }
 528 EXPORT_SYMBOL(__send_remote_softirq);
 529
 530 /**
 531  * send_remote_softirq - try to schedule softirq work on a remote cpu
 532  * @cp: private SMP call function data area
 533  * @cpu: the remote cpu
 534  * @softirq: the softirq for the work
 535  *
 536  * Like __send_remote_softirq except that disabling interrupts and
 537  * computing the current cpu is done for the caller.
 538  */
 539 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 540 {
 541         unsigned long flags;
 542         int this_cpu;
 543
 544         local_irq_save(flags);
 545         this_cpu = smp_processor_id();
 546         __send_remote_softirq(cp, cpu, this_cpu, softirq);
 547         local_irq_restore(flags);
 548 }
 549 EXPORT_SYMBOL(send_remote_softirq);
 550
 551 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
 552                                                unsigned long action, void *hcpu)
 553 {
 554         /*
 555          * If a CPU goes away, splice its entries to the current CPU
 556          * and trigger a run of the softirq
 557          */
 558         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
 559                 int cpu = (unsigned long) hcpu;
 560                 int i;
 561
 562                 local_irq_disable();
 563                 for (i = 0; i < NR_SOFTIRQS; i++) {
 564                         struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
 565                         struct list_head *local_head;
 566
 567                         if (list_empty(head))
 568                                 continue;
 569
 570                         local_head = &__get_cpu_var(softirq_work_list[i]);
 571                         list_splice_init(head, local_head);
 572                         raise_softirq_irqoff(i);
 573                 }
 574                 local_irq_enable();
 575         }
 576
 577         return NOTIFY_OK;
 578 }
 579
 580 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
 581         .notifier_call  = remote_softirq_cpu_notify,
 582 };
 583
 584 void __init softirq_init(void)
 585 {
 586         int cpu;
 587
 588         for_each_possible_cpu(cpu) {
 589                 int i;
 590
 591                 per_cpu(tasklet_vec, cpu).tail =
 592                         &per_cpu(tasklet_vec, cpu).head;
 593                 per_cpu(tasklet_hi_vec, cpu).tail =
 594                         &per_cpu(tasklet_hi_vec, cpu).head;
 595                 for (i = 0; i < NR_SOFTIRQS; i++)
 596                         INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
 597         }
 598
 599         register_hotcpu_notifier(&remote_softirq_cpu_notifier);
 600
 601         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 602         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 603 }
 604
 605 static int ksoftirqd(void * __bind_cpu)
 606 {
 607         set_current_state(TASK_INTERRUPTIBLE);
 608
 609         while (!kthread_should_stop()) {
 610                 preempt_disable();
 611                 if (!local_softirq_pending()) {
 612                         preempt_enable_no_resched();
 613                         schedule();
 614                         preempt_disable();
 615                 }
 616
 617                 __set_current_state(TASK_RUNNING);
 618
 619                 while (local_softirq_pending()) {
 620                         /* Preempt disable stops cpu going offline.
 621                            If already offline, we'll be on wrong CPU:
 622                            don't process */
 623                         if (cpu_is_offline((long)__bind_cpu))
 624                                 goto wait_to_die;
 625                         do_softirq();
 626                         preempt_enable_no_resched();
 627                         cond_resched();
 628                         preempt_disable();
 629                 }
 630                 preempt_enable();
 631                 set_current_state(TASK_INTERRUPTIBLE);
 632         }
 633         __set_current_state(TASK_RUNNING);
 634         return 0;
 635
 636 wait_to_die:
 637         preempt_enable();
 638         /* Wait for kthread_stop */
 639         set_current_state(TASK_INTERRUPTIBLE);
 640         while (!kthread_should_stop()) {
 641                 schedule();
 642                 set_current_state(TASK_INTERRUPTIBLE);
 643         }
 644         __set_current_state(TASK_RUNNING);
 645         return 0;
 646 }
 647
 648 #ifdef CONFIG_HOTPLUG_CPU
 649 /*
 650  * tasklet_kill_immediate is called to remove a tasklet which can already be
 651  * scheduled for execution on @cpu.
 652  *
 653  * Unlike tasklet_kill, this function removes the tasklet
 654  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 655  *
 656  * When this function is called, @cpu must be in the CPU_DEAD state.
 657  */
 658 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 659 {
 660         struct tasklet_struct **i;
 661
 662         BUG_ON(cpu_online(cpu));
 663         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 664
 665         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 666                 return;
 667
 668         /* CPU is dead, so no lock needed. */
 669         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 670                 if (*i == t) {
 671                         *i = t->next;
 672                         /* If this was the tail element, move the tail ptr */
 673                         if (*i == NULL)
 674                                 per_cpu(tasklet_vec, cpu).tail = i;
 675                         return;
 676                 }
 677         }
 678         BUG();
 679 }
 680
 681 static void takeover_tasklets(unsigned int cpu)
 682 {
 683         /* CPU is dead, so no lock needed. */
 684         local_irq_disable();
 685
 686         /* Find end, append list for that CPU. */
 687         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 688                 *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
 689                 __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
 690                 per_cpu(tasklet_vec, cpu).head = NULL;
 691                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 692         }
 693         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 694
 695         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 696                 *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
 697                 __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
 698                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
 699                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 700         }
 701         raise_softirq_irqoff(HI_SOFTIRQ);
 702
 703         local_irq_enable();
 704 }
 705 #endif /* CONFIG_HOTPLUG_CPU */
 706
 707 static int __cpuinit cpu_callback(struct notifier_block *nfb,
 708                                   unsigned long action,
 709                                   void *hcpu)
 710 {
 711         int hotcpu = (unsigned long)hcpu;
 712         struct task_struct *p;
 713
 714         switch (action) {
 715         case CPU_UP_PREPARE:
 716         case CPU_UP_PREPARE_FROZEN:
 717                 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
 718                 if (IS_ERR(p)) {
 719                         printk("ksoftirqd for %i failed\n", hotcpu);
 720                         return NOTIFY_BAD;
 721                 }
 722                 kthread_bind(p, hotcpu);
 723                 per_cpu(ksoftirqd, hotcpu) = p;
 724                 break;
 725         case CPU_ONLINE:
 726         case CPU_ONLINE_FROZEN:
 727                 wake_up_process(per_cpu(ksoftirqd, hotcpu));
 728                 break;
 729 #ifdef CONFIG_HOTPLUG_CPU
 730         case CPU_UP_CANCELED:
 731         case CPU_UP_CANCELED_FROZEN:
 732                 if (!per_cpu(ksoftirqd, hotcpu))
 733                         break;
 734                 /* Unbind so it can run.  Fall thru. */
 735                 kthread_bind(per_cpu(ksoftirqd, hotcpu),
 736                              any_online_cpu(cpu_online_map));
 737         case CPU_DEAD:
 738         case CPU_DEAD_FROZEN: {
 739                 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 740
 741                 p = per_cpu(ksoftirqd, hotcpu);
 742                 per_cpu(ksoftirqd, hotcpu) = NULL;
 743                 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
 744                 kthread_stop(p);
 745                 takeover_tasklets(hotcpu);
 746                 break;
 747         }
 748 #endif /* CONFIG_HOTPLUG_CPU */
 749         }
 750         return NOTIFY_OK;
 751 }
 752
 753 static struct notifier_block __cpuinitdata cpu_nfb = {
 754         .notifier_call = cpu_callback
 755 };
 756
 757 static __init int spawn_ksoftirqd(void)
 758 {
 759         void *cpu = (void *)(long)smp_processor_id();
 760         int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
 761
 762         BUG_ON(err == NOTIFY_BAD);
 763         cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
 764         register_cpu_notifier(&cpu_nfb);
 765         return 0;
 766 }
 767 early_initcall(spawn_ksoftirqd);
 768
 769 #ifdef CONFIG_SMP
 770 /*
 771  * Call a function on all processors
 772  */
 773 int on_each_cpu(void (*func) (void *info), void *info, int wait)
 774 {
 775         int ret = 0;
 776
 777         preempt_disable();
 778         ret = smp_call_function(func, info, wait);
 779         local_irq_disable();
 780         func(info);
 781         local_irq_enable();
 782         preempt_enable();
 783         return ret;
 784 }
 785 EXPORT_SYMBOL(on_each_cpu);
 786 #endif