kernel/time/tick-broadcast.c

   1 /*
   2  * linux/kernel/time/tick-broadcast.c
   3  *
   4  * This file contains functions which emulate a local clock-event
   5  * device via a broadcast event source.
   6  *
   7  * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
   8  * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
   9  * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
  10  *
  11  * This code is licenced under the GPL version 2. For details see
  12  * kernel-base/COPYING.
  13  */
  14 #include <linux/cpu.h>
  15 #include <linux/err.h>
  16 #include <linux/hrtimer.h>
  17 #include <linux/irq.h>
  18 #include <linux/percpu.h>
  19 #include <linux/profile.h>
  20 #include <linux/sched.h>
  21 #include <linux/tick.h>
  22
  23 #include "tick-internal.h"
  24
  25 /*
  26  * Broadcast support for broken x86 hardware, where the local apic
  27  * timer stops in C3 state.
  28  */
  29
  30 struct tick_device tick_broadcast_device;
  31 static cpumask_t tick_broadcast_mask;
  32 static DEFINE_SPINLOCK(tick_broadcast_lock);
  33
  34 /*
  35  * Debugging: see timer_list.c
  36  */
  37 struct tick_device *tick_get_broadcast_device(void)
  38 {
  39         return &tick_broadcast_device;
  40 }
  41
  42 cpumask_t *tick_get_broadcast_mask(void)
  43 {
  44         return &tick_broadcast_mask;
  45 }
  46
  47 /*
  48  * Start the device in periodic mode
  49  */
  50 static void tick_broadcast_start_periodic(struct clock_event_device *bc)
  51 {
  52         if (bc && bc->mode == CLOCK_EVT_MODE_SHUTDOWN)
  53                 tick_setup_periodic(bc, 1);
  54 }
  55
  56 /*
  57  * Check, if the device can be utilized as broadcast device:
  58  */
  59 int tick_check_broadcast_device(struct clock_event_device *dev)
  60 {
  61         if (tick_broadcast_device.evtdev ||
  62             (dev->features & CLOCK_EVT_FEAT_C3STOP))
  63                 return 0;
  64
  65         clockevents_exchange_device(NULL, dev);
  66         tick_broadcast_device.evtdev = dev;
  67         if (!cpus_empty(tick_broadcast_mask))
  68                 tick_broadcast_start_periodic(dev);
  69         return 1;
  70 }
  71
  72 /*
  73  * Check, if the device is the broadcast device
  74  */
  75 int tick_is_broadcast_device(struct clock_event_device *dev)
  76 {
  77         return (dev && tick_broadcast_device.evtdev == dev);
  78 }
  79
  80 /*
  81  * Check, if the device is disfunctional and a place holder, which
  82  * needs to be handled by the broadcast device.
  83  */
  84 int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
  85 {
  86         unsigned long flags;
  87         int ret = 0;
  88
  89         spin_lock_irqsave(&tick_broadcast_lock, flags);
  90
  91         /*
  92          * Devices might be registered with both periodic and oneshot
  93          * mode disabled. This signals, that the device needs to be
  94          * operated from the broadcast device and is a placeholder for
  95          * the cpu local device.
  96          */
  97         if (!tick_device_is_functional(dev)) {
  98                 dev->event_handler = tick_handle_periodic;
  99                 cpu_set(cpu, tick_broadcast_mask);
 100                 tick_broadcast_start_periodic(tick_broadcast_device.evtdev);
 101                 ret = 1;
 102         }
 103
 104         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 105         return ret;
 106 }
 107
 108 /*
 109  * Broadcast the event to the cpus, which are set in the mask
 110  */
 111 int tick_do_broadcast(cpumask_t mask)
 112 {
 113         int ret = 0, cpu = smp_processor_id();
 114         struct tick_device *td;
 115
 116         /*
 117          * Check, if the current cpu is in the mask
 118          */
 119         if (cpu_isset(cpu, mask)) {
 120                 cpu_clear(cpu, mask);
 121                 td = &per_cpu(tick_cpu_device, cpu);
 122                 td->evtdev->event_handler(td->evtdev);
 123                 ret = 1;
 124         }
 125
 126         if (!cpus_empty(mask)) {
 127                 /*
 128                  * It might be necessary to actually check whether the devices
 129                  * have different broadcast functions. For now, just use the
 130                  * one of the first device. This works as long as we have this
 131                  * misfeature only on x86 (lapic)
 132                  */
 133                 cpu = first_cpu(mask);
 134                 td = &per_cpu(tick_cpu_device, cpu);
 135                 td->evtdev->broadcast(mask);
 136                 ret = 1;
 137         }
 138         return ret;
 139 }
 140
 141 /*
 142  * Periodic broadcast:
 143  * - invoke the broadcast handlers
 144  */
 145 static void tick_do_periodic_broadcast(void)
 146 {
 147         cpumask_t mask;
 148
 149         spin_lock(&tick_broadcast_lock);
 150
 151         cpus_and(mask, cpu_online_map, tick_broadcast_mask);
 152         tick_do_broadcast(mask);
 153
 154         spin_unlock(&tick_broadcast_lock);
 155 }
 156
 157 /*
 158  * Event handler for periodic broadcast ticks
 159  */
 160 static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
 161 {
 162         dev->next_event.tv64 = KTIME_MAX;
 163
 164         tick_do_periodic_broadcast();
 165
 166         /*
 167          * The device is in periodic mode. No reprogramming necessary:
 168          */
 169         if (dev->mode == CLOCK_EVT_MODE_PERIODIC)
 170                 return;
 171
 172         /*
 173          * Setup the next period for devices, which do not have
 174          * periodic mode:
 175          */
 176         for (;;) {
 177                 ktime_t next = ktime_add(dev->next_event, tick_period);
 178
 179                 if (!clockevents_program_event(dev, next, ktime_get()))
 180                         return;
 181                 tick_do_periodic_broadcast();
 182         }
 183 }
 184
 185 /*
 186  * Powerstate information: The system enters/leaves a state, where
 187  * affected devices might stop
 188  */
 189 static void tick_do_broadcast_on_off(void *why)
 190 {
 191         struct clock_event_device *bc, *dev;
 192         struct tick_device *td;
 193         unsigned long flags, *reason = why;
 194         int cpu;
 195
 196         spin_lock_irqsave(&tick_broadcast_lock, flags);
 197
 198         cpu = smp_processor_id();
 199         td = &per_cpu(tick_cpu_device, cpu);
 200         dev = td->evtdev;
 201         bc = tick_broadcast_device.evtdev;
 202
 203         /*
 204          * Is the device in broadcast mode forever or is it not
 205          * affected by the powerstate ?
 206          */
 207         if (!dev || !tick_device_is_functional(dev) ||
 208             !(dev->features & CLOCK_EVT_FEAT_C3STOP))
 209                 goto out;
 210
 211         if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_ON) {
 212                 if (!cpu_isset(cpu, tick_broadcast_mask)) {
 213                         cpu_set(cpu, tick_broadcast_mask);
 214                         if (td->mode == TICKDEV_MODE_PERIODIC)
 215                                 clockevents_set_mode(dev,
 216                                                      CLOCK_EVT_MODE_SHUTDOWN);
 217                 }
 218         } else {
 219                 if (cpu_isset(cpu, tick_broadcast_mask)) {
 220                         cpu_clear(cpu, tick_broadcast_mask);
 221                         if (td->mode == TICKDEV_MODE_PERIODIC)
 222                                 tick_setup_periodic(dev, 0);
 223                 }
 224         }
 225
 226         if (cpus_empty(tick_broadcast_mask))
 227                 clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN);
 228         else {
 229                 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
 230                         tick_broadcast_start_periodic(bc);
 231                 else
 232                         tick_broadcast_setup_oneshot(bc);
 233         }
 234 out:
 235         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 236 }
 237
 238 /*
 239  * Powerstate information: The system enters/leaves a state, where
 240  * affected devices might stop.
 241  */
 242 void tick_broadcast_on_off(unsigned long reason, int *oncpu)
 243 {
 244         int cpu = get_cpu();
 245
 246         if (cpu == *oncpu)
 247                 tick_do_broadcast_on_off(&reason);
 248         else
 249                 smp_call_function_single(*oncpu, tick_do_broadcast_on_off,
 250                                          &reason, 1, 1);
 251         put_cpu();
 252 }
 253
 254 /*
 255  * Set the periodic handler depending on broadcast on/off
 256  */
 257 void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast)
 258 {
 259         if (!broadcast)
 260                 dev->event_handler = tick_handle_periodic;
 261         else
 262                 dev->event_handler = tick_handle_periodic_broadcast;
 263 }
 264
 265 /*
 266  * Remove a CPU from broadcasting
 267  */
 268 void tick_shutdown_broadcast(unsigned int *cpup)
 269 {
 270         struct clock_event_device *bc;
 271         unsigned long flags;
 272         unsigned int cpu = *cpup;
 273
 274         spin_lock_irqsave(&tick_broadcast_lock, flags);
 275
 276         bc = tick_broadcast_device.evtdev;
 277         cpu_clear(cpu, tick_broadcast_mask);
 278
 279         if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
 280                 if (bc && cpus_empty(tick_broadcast_mask))
 281                         clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN);
 282         }
 283
 284         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 285 }
 286
 287 void tick_suspend_broadcast(void)
 288 {
 289         struct clock_event_device *bc;
 290         unsigned long flags;
 291
 292         spin_lock_irqsave(&tick_broadcast_lock, flags);
 293
 294         bc = tick_broadcast_device.evtdev;
 295         if (bc && tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
 296                 clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN);
 297
 298         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 299 }
 300
 301 int tick_resume_broadcast(void)
 302 {
 303         struct clock_event_device *bc;
 304         unsigned long flags;
 305         int broadcast = 0;
 306
 307         spin_lock_irqsave(&tick_broadcast_lock, flags);
 308
 309         bc = tick_broadcast_device.evtdev;
 310
 311         if (bc) {
 312                 switch (tick_broadcast_device.mode) {
 313                 case TICKDEV_MODE_PERIODIC:
 314                         if(!cpus_empty(tick_broadcast_mask))
 315                                 tick_broadcast_start_periodic(bc);
 316                         broadcast = cpu_isset(smp_processor_id(),
 317                                               tick_broadcast_mask);
 318                         break;
 319                 case TICKDEV_MODE_ONESHOT:
 320                         broadcast = tick_resume_broadcast_oneshot(bc);
 321                         break;
 322                 }
 323         }
 324         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 325
 326         return broadcast;
 327 }
 328
 329
 330 #ifdef CONFIG_TICK_ONESHOT
 331
 332 static cpumask_t tick_broadcast_oneshot_mask;
 333
 334 /*
 335  * Debugging: see timer_list.c
 336  */
 337 cpumask_t *tick_get_broadcast_oneshot_mask(void)
 338 {
 339         return &tick_broadcast_oneshot_mask;
 340 }
 341
 342 static int tick_broadcast_set_event(ktime_t expires, int force)
 343 {
 344         struct clock_event_device *bc = tick_broadcast_device.evtdev;
 345         ktime_t now = ktime_get();
 346         int res;
 347
 348         for(;;) {
 349                 res = clockevents_program_event(bc, expires, now);
 350                 if (!res || !force)
 351                         return res;
 352                 now = ktime_get();
 353                 expires = ktime_add(now, ktime_set(0, bc->min_delta_ns));
 354         }
 355 }
 356
 357 int tick_resume_broadcast_oneshot(struct clock_event_device *bc)
 358 {
 359         clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
 360
 361         if(!cpus_empty(tick_broadcast_oneshot_mask))
 362                 tick_broadcast_set_event(ktime_get(), 1);
 363
 364         return cpu_isset(smp_processor_id(), tick_broadcast_oneshot_mask);
 365 }
 366
 367 /*
 368  * Reprogram the broadcast device:
 369  *
 370  * Called with tick_broadcast_lock held and interrupts disabled.
 371  */
 372 static int tick_broadcast_reprogram(void)
 373 {
 374         ktime_t expires = { .tv64 = KTIME_MAX };
 375         struct tick_device *td;
 376         int cpu;
 377
 378         /*
 379          * Find the event which expires next:
 380          */
 381         for (cpu = first_cpu(tick_broadcast_oneshot_mask); cpu != NR_CPUS;
 382              cpu = next_cpu(cpu, tick_broadcast_oneshot_mask)) {
 383                 td = &per_cpu(tick_cpu_device, cpu);
 384                 if (td->evtdev->next_event.tv64 < expires.tv64)
 385                         expires = td->evtdev->next_event;
 386         }
 387
 388         if (expires.tv64 == KTIME_MAX)
 389                 return 0;
 390
 391         return tick_broadcast_set_event(expires, 0);
 392 }
 393
 394 /*
 395  * Handle oneshot mode broadcasting
 396  */
 397 static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
 398 {
 399         struct tick_device *td;
 400         cpumask_t mask;
 401         ktime_t now;
 402         int cpu;
 403
 404         spin_lock(&tick_broadcast_lock);
 405 again:
 406         dev->next_event.tv64 = KTIME_MAX;
 407         mask = CPU_MASK_NONE;
 408         now = ktime_get();
 409         /* Find all expired events */
 410         for (cpu = first_cpu(tick_broadcast_oneshot_mask); cpu != NR_CPUS;
 411              cpu = next_cpu(cpu, tick_broadcast_oneshot_mask)) {
 412                 td = &per_cpu(tick_cpu_device, cpu);
 413                 if (td->evtdev->next_event.tv64 <= now.tv64)
 414                         cpu_set(cpu, mask);
 415         }
 416
 417         /*
 418          * Wakeup the cpus which have an expired event. The broadcast
 419          * device is reprogrammed in the return from idle code.
 420          */
 421         if (!tick_do_broadcast(mask)) {
 422                 /*
 423                  * The global event did not expire any CPU local
 424                  * events. This happens in dyntick mode, as the
 425                  * maximum PIT delta is quite small.
 426                  */
 427                 if (tick_broadcast_reprogram())
 428                         goto again;
 429         }
 430         spin_unlock(&tick_broadcast_lock);
 431 }
 432
 433 /*
 434  * Powerstate information: The system enters/leaves a state, where
 435  * affected devices might stop
 436  */
 437 void tick_broadcast_oneshot_control(unsigned long reason)
 438 {
 439         struct clock_event_device *bc, *dev;
 440         struct tick_device *td;
 441         unsigned long flags;
 442         int cpu;
 443
 444         spin_lock_irqsave(&tick_broadcast_lock, flags);
 445
 446         /*
 447          * Periodic mode does not care about the enter/exit of power
 448          * states
 449          */
 450         if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
 451                 goto out;
 452
 453         bc = tick_broadcast_device.evtdev;
 454         cpu = smp_processor_id();
 455         td = &per_cpu(tick_cpu_device, cpu);
 456         dev = td->evtdev;
 457
 458         if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
 459                 goto out;
 460
 461         if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) {
 462                 if (!cpu_isset(cpu, tick_broadcast_oneshot_mask)) {
 463                         cpu_set(cpu, tick_broadcast_oneshot_mask);
 464                         clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN);
 465                         if (dev->next_event.tv64 < bc->next_event.tv64)
 466                                 tick_broadcast_set_event(dev->next_event, 1);
 467                 }
 468         } else {
 469                 if (cpu_isset(cpu, tick_broadcast_oneshot_mask)) {
 470                         cpu_clear(cpu, tick_broadcast_oneshot_mask);
 471                         clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT);
 472                         if (dev->next_event.tv64 != KTIME_MAX)
 473                                 tick_program_event(dev->next_event, 1);
 474                 }
 475         }
 476
 477 out:
 478         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 479 }
 480
 481 /**
 482  * tick_broadcast_setup_highres - setup the broadcast device for highres
 483  */
 484 void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
 485 {
 486         if (bc->mode != CLOCK_EVT_MODE_ONESHOT) {
 487                 bc->event_handler = tick_handle_oneshot_broadcast;
 488                 clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
 489                 bc->next_event.tv64 = KTIME_MAX;
 490         }
 491 }
 492
 493 /*
 494  * Select oneshot operating mode for the broadcast device
 495  */
 496 void tick_broadcast_switch_to_oneshot(void)
 497 {
 498         struct clock_event_device *bc;
 499         unsigned long flags;
 500
 501         spin_lock_irqsave(&tick_broadcast_lock, flags);
 502
 503         tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
 504         bc = tick_broadcast_device.evtdev;
 505         if (bc)
 506                 tick_broadcast_setup_oneshot(bc);
 507         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 508 }
 509
 510
 511 /*
 512  * Remove a dead CPU from broadcasting
 513  */
 514 void tick_shutdown_broadcast_oneshot(unsigned int *cpup)
 515 {
 516         struct clock_event_device *bc;
 517         unsigned long flags;
 518         unsigned int cpu = *cpup;
 519
 520         spin_lock_irqsave(&tick_broadcast_lock, flags);
 521
 522         bc = tick_broadcast_device.evtdev;
 523         cpu_clear(cpu, tick_broadcast_oneshot_mask);
 524
 525         if (tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT) {
 526                 if (bc && cpus_empty(tick_broadcast_oneshot_mask))
 527                         clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN);
 528         }
 529
 530         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 531 }
 532
 533 #endif