kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   7  */
   8
   9 #include <linux/module.h>
  10 #include <linux/kernel_stat.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/init.h>
  13 #include <linux/mm.h>
  14 #include <linux/notifier.h>
  15 #include <linux/percpu.h>
  16 #include <linux/cpu.h>
  17 #include <linux/kthread.h>
  18 #include <linux/rcupdate.h>
  19 #include <linux/smp.h>
  20
  21 #include <asm/irq.h>
  22 /*
  23    - No shared variables, all the data are CPU local.
  24    - If a softirq needs serialization, let it serialize itself
  25      by its own spinlocks.
  26    - Even if softirq is serialized, only local cpu is marked for
  27      execution. Hence, we get something sort of weak cpu binding.
  28      Though it is still not clear, will it result in better locality
  29      or will not.
  30
  31    Examples:
  32    - NET RX softirq. It is multithreaded and does not require
  33      any global serialization.
  34    - NET TX softirq. It kicks software netdevice queues, hence
  35      it is logically serialized per device, but this serialization
  36      is invisible to common code.
  37    - Tasklets: serialized wrt itself.
  38  */
  39
  40 #ifndef __ARCH_IRQ_STAT
  41 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  42 EXPORT_SYMBOL(irq_stat);
  43 #endif
  44
  45 static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
  46
  47 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  48
  49 /*
  50  * we cannot loop indefinitely here to avoid userspace starvation,
  51  * but we also don't want to introduce a worst case 1/HZ latency
  52  * to the pending events, so lets the scheduler to balance
  53  * the softirq load for us.
  54  */
  55 static inline void wakeup_softirqd(void)
  56 {
  57         /* Interrupts are disabled: no need to stop preemption */
  58         struct task_struct *tsk = __get_cpu_var(ksoftirqd);
  59
  60         if (tsk && tsk->state != TASK_RUNNING)
  61                 wake_up_process(tsk);
  62 }
  63
  64 /*
  65  * This one is for softirq.c-internal use,
  66  * where hardirqs are disabled legitimately:
  67  */
  68 #ifdef CONFIG_TRACE_IRQFLAGS
  69 static void __local_bh_disable(unsigned long ip)
  70 {
  71         unsigned long flags;
  72
  73         WARN_ON_ONCE(in_irq());
  74
  75         raw_local_irq_save(flags);
  76         add_preempt_count(SOFTIRQ_OFFSET);
  77         /*
  78          * Were softirqs turned off above:
  79          */
  80         if (softirq_count() == SOFTIRQ_OFFSET)
  81                 trace_softirqs_off(ip);
  82         raw_local_irq_restore(flags);
  83 }
  84 #else /* !CONFIG_TRACE_IRQFLAGS */
  85 static inline void __local_bh_disable(unsigned long ip)
  86 {
  87         add_preempt_count(SOFTIRQ_OFFSET);
  88         barrier();
  89 }
  90 #endif /* CONFIG_TRACE_IRQFLAGS */
  91
  92 void local_bh_disable(void)
  93 {
  94         __local_bh_disable((unsigned long)__builtin_return_address(0));
  95 }
  96
  97 EXPORT_SYMBOL(local_bh_disable);
  98
  99 void __local_bh_enable(void)
 100 {
 101         WARN_ON_ONCE(in_irq());
 102
 103         /*
 104          * softirqs should never be enabled by __local_bh_enable(),
 105          * it always nests inside local_bh_enable() sections:
 106          */
 107         WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
 108
 109         sub_preempt_count(SOFTIRQ_OFFSET);
 110 }
 111 EXPORT_SYMBOL_GPL(__local_bh_enable);
 112
 113 /*
 114  * Special-case - softirqs can safely be enabled in
 115  * cond_resched_softirq(), or by __do_softirq(),
 116  * without processing still-pending softirqs:
 117  */
 118 void _local_bh_enable(void)
 119 {
 120         WARN_ON_ONCE(in_irq());
 121         WARN_ON_ONCE(!irqs_disabled());
 122
 123         if (softirq_count() == SOFTIRQ_OFFSET)
 124                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 125         sub_preempt_count(SOFTIRQ_OFFSET);
 126 }
 127
 128 EXPORT_SYMBOL(_local_bh_enable);
 129
 130 void local_bh_enable(void)
 131 {
 132 #ifdef CONFIG_TRACE_IRQFLAGS
 133         unsigned long flags;
 134
 135         WARN_ON_ONCE(in_irq());
 136 #endif
 137         WARN_ON_ONCE(irqs_disabled());
 138
 139 #ifdef CONFIG_TRACE_IRQFLAGS
 140         local_irq_save(flags);
 141 #endif
 142         /*
 143          * Are softirqs going to be turned on now:
 144          */
 145         if (softirq_count() == SOFTIRQ_OFFSET)
 146                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 147         /*
 148          * Keep preemption disabled until we are done with
 149          * softirq processing:
 150          */
 151         sub_preempt_count(SOFTIRQ_OFFSET - 1);
 152
 153         if (unlikely(!in_interrupt() && local_softirq_pending()))
 154                 do_softirq();
 155
 156         dec_preempt_count();
 157 #ifdef CONFIG_TRACE_IRQFLAGS
 158         local_irq_restore(flags);
 159 #endif
 160         preempt_check_resched();
 161 }
 162 EXPORT_SYMBOL(local_bh_enable);
 163
 164 void local_bh_enable_ip(unsigned long ip)
 165 {
 166 #ifdef CONFIG_TRACE_IRQFLAGS
 167         unsigned long flags;
 168
 169         WARN_ON_ONCE(in_irq());
 170
 171         local_irq_save(flags);
 172 #endif
 173         /*
 174          * Are softirqs going to be turned on now:
 175          */
 176         if (softirq_count() == SOFTIRQ_OFFSET)
 177                 trace_softirqs_on(ip);
 178         /*
 179          * Keep preemption disabled until we are done with
 180          * softirq processing:
 181          */
 182         sub_preempt_count(SOFTIRQ_OFFSET - 1);
 183
 184         if (unlikely(!in_interrupt() && local_softirq_pending()))
 185                 do_softirq();
 186
 187         dec_preempt_count();
 188 #ifdef CONFIG_TRACE_IRQFLAGS
 189         local_irq_restore(flags);
 190 #endif
 191         preempt_check_resched();
 192 }
 193 EXPORT_SYMBOL(local_bh_enable_ip);
 194
 195 /*
 196  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
 197  * and we fall back to softirqd after that.
 198  *
 199  * This number has been established via experimentation.
 200  * The two things to balance is latency against fairness -
 201  * we want to handle softirqs as soon as possible, but they
 202  * should not be able to lock up the box.
 203  */
 204 #define MAX_SOFTIRQ_RESTART 10
 205
 206 asmlinkage void __do_softirq(void)
 207 {
 208         struct softirq_action *h;
 209         __u32 pending;
 210         int max_restart = MAX_SOFTIRQ_RESTART;
 211         int cpu;
 212
 213         pending = local_softirq_pending();
 214         account_system_vtime(current);
 215
 216         __local_bh_disable((unsigned long)__builtin_return_address(0));
 217         trace_softirq_enter();
 218
 219         cpu = smp_processor_id();
 220 restart:
 221         /* Reset the pending bitmask before enabling irqs */
 222         set_softirq_pending(0);
 223
 224         local_irq_enable();
 225
 226         h = softirq_vec;
 227
 228         do {
 229                 if (pending & 1) {
 230                         h->action(h);
 231                         rcu_bh_qsctr_inc(cpu);
 232                 }
 233                 h++;
 234                 pending >>= 1;
 235         } while (pending);
 236
 237         local_irq_disable();
 238
 239         pending = local_softirq_pending();
 240         if (pending && --max_restart)
 241                 goto restart;
 242
 243         if (pending)
 244                 wakeup_softirqd();
 245
 246         trace_softirq_exit();
 247
 248         account_system_vtime(current);
 249         _local_bh_enable();
 250 }
 251
 252 #ifndef __ARCH_HAS_DO_SOFTIRQ
 253
 254 asmlinkage void do_softirq(void)
 255 {
 256         __u32 pending;
 257         unsigned long flags;
 258
 259         if (in_interrupt())
 260                 return;
 261
 262         local_irq_save(flags);
 263
 264         pending = local_softirq_pending();
 265
 266         if (pending)
 267                 __do_softirq();
 268
 269         local_irq_restore(flags);
 270 }
 271
 272 EXPORT_SYMBOL(do_softirq);
 273
 274 #endif
 275
 276 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
 277 # define invoke_softirq()       __do_softirq()
 278 #else
 279 # define invoke_softirq()       do_softirq()
 280 #endif
 281
 282 /*
 283  * Exit an interrupt context. Process softirqs if needed and possible:
 284  */
 285 void irq_exit(void)
 286 {
 287         account_system_vtime(current);
 288         trace_hardirq_exit();
 289         sub_preempt_count(IRQ_EXIT_OFFSET);
 290         if (!in_interrupt() && local_softirq_pending())
 291                 invoke_softirq();
 292         preempt_enable_no_resched();
 293 }
 294
 295 /*
 296  * This function must run with irqs disabled!
 297  */
 298 inline fastcall void raise_softirq_irqoff(unsigned int nr)
 299 {
 300         __raise_softirq_irqoff(nr);
 301
 302         /*
 303          * If we're in an interrupt or softirq, we're done
 304          * (this also catches softirq-disabled code). We will
 305          * actually run the softirq once we return from
 306          * the irq or softirq.
 307          *
 308          * Otherwise we wake up ksoftirqd to make sure we
 309          * schedule the softirq soon.
 310          */
 311         if (!in_interrupt())
 312                 wakeup_softirqd();
 313 }
 314
 315 EXPORT_SYMBOL(raise_softirq_irqoff);
 316
 317 void fastcall raise_softirq(unsigned int nr)
 318 {
 319         unsigned long flags;
 320
 321         local_irq_save(flags);
 322         raise_softirq_irqoff(nr);
 323         local_irq_restore(flags);
 324 }
 325
 326 void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
 327 {
 328         softirq_vec[nr].data = data;
 329         softirq_vec[nr].action = action;
 330 }
 331
 332 /* Tasklets */
 333 struct tasklet_head
 334 {
 335         struct tasklet_struct *list;
 336 };
 337
 338 /* Some compilers disobey section attribute on statics when not
 339    initialized -- RR */
 340 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
 341 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
 342
 343 void fastcall __tasklet_schedule(struct tasklet_struct *t)
 344 {
 345         unsigned long flags;
 346
 347         local_irq_save(flags);
 348         t->next = __get_cpu_var(tasklet_vec).list;
 349         __get_cpu_var(tasklet_vec).list = t;
 350         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 351         local_irq_restore(flags);
 352 }
 353
 354 EXPORT_SYMBOL(__tasklet_schedule);
 355
 356 void fastcall __tasklet_hi_schedule(struct tasklet_struct *t)
 357 {
 358         unsigned long flags;
 359
 360         local_irq_save(flags);
 361         t->next = __get_cpu_var(tasklet_hi_vec).list;
 362         __get_cpu_var(tasklet_hi_vec).list = t;
 363         raise_softirq_irqoff(HI_SOFTIRQ);
 364         local_irq_restore(flags);
 365 }
 366
 367 EXPORT_SYMBOL(__tasklet_hi_schedule);
 368
 369 static void tasklet_action(struct softirq_action *a)
 370 {
 371         struct tasklet_struct *list;
 372
 373         local_irq_disable();
 374         list = __get_cpu_var(tasklet_vec).list;
 375         __get_cpu_var(tasklet_vec).list = NULL;
 376         local_irq_enable();
 377
 378         while (list) {
 379                 struct tasklet_struct *t = list;
 380
 381                 list = list->next;
 382
 383                 if (tasklet_trylock(t)) {
 384                         if (!atomic_read(&t->count)) {
 385                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 386                                         BUG();
 387                                 t->func(t->data);
 388                                 tasklet_unlock(t);
 389                                 continue;
 390                         }
 391                         tasklet_unlock(t);
 392                 }
 393
 394                 local_irq_disable();
 395                 t->next = __get_cpu_var(tasklet_vec).list;
 396                 __get_cpu_var(tasklet_vec).list = t;
 397                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 398                 local_irq_enable();
 399         }
 400 }
 401
 402 static void tasklet_hi_action(struct softirq_action *a)
 403 {
 404         struct tasklet_struct *list;
 405
 406         local_irq_disable();
 407         list = __get_cpu_var(tasklet_hi_vec).list;
 408         __get_cpu_var(tasklet_hi_vec).list = NULL;
 409         local_irq_enable();
 410
 411         while (list) {
 412                 struct tasklet_struct *t = list;
 413
 414                 list = list->next;
 415
 416                 if (tasklet_trylock(t)) {
 417                         if (!atomic_read(&t->count)) {
 418                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 419                                         BUG();
 420                                 t->func(t->data);
 421                                 tasklet_unlock(t);
 422                                 continue;
 423                         }
 424                         tasklet_unlock(t);
 425                 }
 426
 427                 local_irq_disable();
 428                 t->next = __get_cpu_var(tasklet_hi_vec).list;
 429                 __get_cpu_var(tasklet_hi_vec).list = t;
 430                 __raise_softirq_irqoff(HI_SOFTIRQ);
 431                 local_irq_enable();
 432         }
 433 }
 434
 435
 436 void tasklet_init(struct tasklet_struct *t,
 437                   void (*func)(unsigned long), unsigned long data)
 438 {
 439         t->next = NULL;
 440         t->state = 0;
 441         atomic_set(&t->count, 0);
 442         t->func = func;
 443         t->data = data;
 444 }
 445
 446 EXPORT_SYMBOL(tasklet_init);
 447
 448 void tasklet_kill(struct tasklet_struct *t)
 449 {
 450         if (in_interrupt())
 451                 printk("Attempt to kill tasklet from interrupt\n");
 452
 453         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 454                 do
 455                         yield();
 456                 while (test_bit(TASKLET_STATE_SCHED, &t->state));
 457         }
 458         tasklet_unlock_wait(t);
 459         clear_bit(TASKLET_STATE_SCHED, &t->state);
 460 }
 461
 462 EXPORT_SYMBOL(tasklet_kill);
 463
 464 void __init softirq_init(void)
 465 {
 466         open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
 467         open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
 468 }
 469
 470 static int ksoftirqd(void * __bind_cpu)
 471 {
 472         set_user_nice(current, 19);
 473         current->flags |= PF_NOFREEZE;
 474
 475         set_current_state(TASK_INTERRUPTIBLE);
 476
 477         while (!kthread_should_stop()) {
 478                 preempt_disable();
 479                 if (!local_softirq_pending()) {
 480                         preempt_enable_no_resched();
 481                         schedule();
 482                         preempt_disable();
 483                 }
 484
 485                 __set_current_state(TASK_RUNNING);
 486
 487                 while (local_softirq_pending()) {
 488                         /* Preempt disable stops cpu going offline.
 489                            If already offline, we'll be on wrong CPU:
 490                            don't process */
 491                         if (cpu_is_offline((long)__bind_cpu))
 492                                 goto wait_to_die;
 493                         do_softirq();
 494                         preempt_enable_no_resched();
 495                         cond_resched();
 496                         preempt_disable();
 497                 }
 498                 preempt_enable();
 499                 set_current_state(TASK_INTERRUPTIBLE);
 500         }
 501         __set_current_state(TASK_RUNNING);
 502         return 0;
 503
 504 wait_to_die:
 505         preempt_enable();
 506         /* Wait for kthread_stop */
 507         set_current_state(TASK_INTERRUPTIBLE);
 508         while (!kthread_should_stop()) {
 509                 schedule();
 510                 set_current_state(TASK_INTERRUPTIBLE);
 511         }
 512         __set_current_state(TASK_RUNNING);
 513         return 0;
 514 }
 515
 516 #ifdef CONFIG_HOTPLUG_CPU
 517 /*
 518  * tasklet_kill_immediate is called to remove a tasklet which can already be
 519  * scheduled for execution on @cpu.
 520  *
 521  * Unlike tasklet_kill, this function removes the tasklet
 522  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 523  *
 524  * When this function is called, @cpu must be in the CPU_DEAD state.
 525  */
 526 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 527 {
 528         struct tasklet_struct **i;
 529
 530         BUG_ON(cpu_online(cpu));
 531         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 532
 533         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 534                 return;
 535
 536         /* CPU is dead, so no lock needed. */
 537         for (i = &per_cpu(tasklet_vec, cpu).list; *i; i = &(*i)->next) {
 538                 if (*i == t) {
 539                         *i = t->next;
 540                         return;
 541                 }
 542         }
 543         BUG();
 544 }
 545
 546 static void takeover_tasklets(unsigned int cpu)
 547 {
 548         struct tasklet_struct **i;
 549
 550         /* CPU is dead, so no lock needed. */
 551         local_irq_disable();
 552
 553         /* Find end, append list for that CPU. */
 554         for (i = &__get_cpu_var(tasklet_vec).list; *i; i = &(*i)->next);
 555         *i = per_cpu(tasklet_vec, cpu).list;
 556         per_cpu(tasklet_vec, cpu).list = NULL;
 557         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 558
 559         for (i = &__get_cpu_var(tasklet_hi_vec).list; *i; i = &(*i)->next);
 560         *i = per_cpu(tasklet_hi_vec, cpu).list;
 561         per_cpu(tasklet_hi_vec, cpu).list = NULL;
 562         raise_softirq_irqoff(HI_SOFTIRQ);
 563
 564         local_irq_enable();
 565 }
 566 #endif /* CONFIG_HOTPLUG_CPU */
 567
 568 static int __cpuinit cpu_callback(struct notifier_block *nfb,
 569                                   unsigned long action,
 570                                   void *hcpu)
 571 {
 572         int hotcpu = (unsigned long)hcpu;
 573         struct task_struct *p;
 574
 575         switch (action) {
 576         case CPU_UP_PREPARE:
 577                 BUG_ON(per_cpu(tasklet_vec, hotcpu).list);
 578                 BUG_ON(per_cpu(tasklet_hi_vec, hotcpu).list);
 579                 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
 580                 if (IS_ERR(p)) {
 581                         printk("ksoftirqd for %i failed\n", hotcpu);
 582                         return NOTIFY_BAD;
 583                 }
 584                 kthread_bind(p, hotcpu);
 585                 per_cpu(ksoftirqd, hotcpu) = p;
 586                 break;
 587         case CPU_ONLINE:
 588                 wake_up_process(per_cpu(ksoftirqd, hotcpu));
 589                 break;
 590 #ifdef CONFIG_HOTPLUG_CPU
 591         case CPU_UP_CANCELED:
 592                 if (!per_cpu(ksoftirqd, hotcpu))
 593                         break;
 594                 /* Unbind so it can run.  Fall thru. */
 595                 kthread_bind(per_cpu(ksoftirqd, hotcpu),
 596                              any_online_cpu(cpu_online_map));
 597         case CPU_DEAD:
 598                 p = per_cpu(ksoftirqd, hotcpu);
 599                 per_cpu(ksoftirqd, hotcpu) = NULL;
 600                 kthread_stop(p);
 601                 takeover_tasklets(hotcpu);
 602                 break;
 603 #endif /* CONFIG_HOTPLUG_CPU */
 604         }
 605         return NOTIFY_OK;
 606 }
 607
 608 static struct notifier_block __cpuinitdata cpu_nfb = {
 609         .notifier_call = cpu_callback
 610 };
 611
 612 __init int spawn_ksoftirqd(void)
 613 {
 614         void *cpu = (void *)(long)smp_processor_id();
 615         int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
 616
 617         BUG_ON(err == NOTIFY_BAD);
 618         cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
 619         register_cpu_notifier(&cpu_nfb);
 620         return 0;
 621 }
 622
 623 #ifdef CONFIG_SMP
 624 /*
 625  * Call a function on all processors
 626  */
 627 int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait)
 628 {
 629         int ret = 0;
 630
 631         preempt_disable();
 632         ret = smp_call_function(func, info, retry, wait);
 633         local_irq_disable();
 634         func(info);
 635         local_irq_enable();
 636         preempt_enable();
 637         return ret;
 638 }
 639 EXPORT_SYMBOL(on_each_cpu);
 640 #endif