kernel/time/tick-broadcast.c

   1 /*
   2  * linux/kernel/time/tick-broadcast.c
   3  *
   4  * This file contains functions which emulate a local clock-event
   5  * device via a broadcast event source.
   6  *
   7  * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
   8  * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
   9  * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
  10  *
  11  * This code is licenced under the GPL version 2. For details see
  12  * kernel-base/COPYING.
  13  */
  14 #include <linux/cpu.h>
  15 #include <linux/err.h>
  16 #include <linux/hrtimer.h>
  17 #include <linux/irq.h>
  18 #include <linux/percpu.h>
  19 #include <linux/profile.h>
  20 #include <linux/sched.h>
  21 #include <linux/tick.h>
  22
  23 #include "tick-internal.h"
  24
  25 /*
  26  * Broadcast support for broken x86 hardware, where the local apic
  27  * timer stops in C3 state.
  28  */
  29
  30 struct tick_device tick_broadcast_device;
  31 static cpumask_t tick_broadcast_mask;
  32 static DEFINE_SPINLOCK(tick_broadcast_lock);
  33
  34 /*
  35  * Debugging: see timer_list.c
  36  */
  37 struct tick_device *tick_get_broadcast_device(void)
  38 {
  39         return &tick_broadcast_device;
  40 }
  41
  42 cpumask_t *tick_get_broadcast_mask(void)
  43 {
  44         return &tick_broadcast_mask;
  45 }
  46
  47 /*
  48  * Start the device in periodic mode
  49  */
  50 static void tick_broadcast_start_periodic(struct clock_event_device *bc)
  51 {
  52         if (bc && bc->mode == CLOCK_EVT_MODE_SHUTDOWN)
  53                 tick_setup_periodic(bc, 1);
  54 }
  55
  56 /*
  57  * Check, if the device can be utilized as broadcast device:
  58  */
  59 int tick_check_broadcast_device(struct clock_event_device *dev)
  60 {
  61         if (tick_broadcast_device.evtdev ||
  62             (dev->features & CLOCK_EVT_FEAT_C3STOP))
  63                 return 0;
  64
  65         clockevents_exchange_device(NULL, dev);
  66         tick_broadcast_device.evtdev = dev;
  67         if (!cpus_empty(tick_broadcast_mask))
  68                 tick_broadcast_start_periodic(dev);
  69         return 1;
  70 }
  71
  72 /*
  73  * Check, if the device is the broadcast device
  74  */
  75 int tick_is_broadcast_device(struct clock_event_device *dev)
  76 {
  77         return (dev && tick_broadcast_device.evtdev == dev);
  78 }
  79
  80 /*
  81  * Check, if the device is disfunctional and a place holder, which
  82  * needs to be handled by the broadcast device.
  83  */
  84 int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
  85 {
  86         unsigned long flags;
  87         int ret = 0;
  88
  89         spin_lock_irqsave(&tick_broadcast_lock, flags);
  90
  91         /*
  92          * Devices might be registered with both periodic and oneshot
  93          * mode disabled. This signals, that the device needs to be
  94          * operated from the broadcast device and is a placeholder for
  95          * the cpu local device.
  96          */
  97         if (!tick_device_is_functional(dev)) {
  98                 dev->event_handler = tick_handle_periodic;
  99                 cpu_set(cpu, tick_broadcast_mask);
 100                 tick_broadcast_start_periodic(tick_broadcast_device.evtdev);
 101                 ret = 1;
 102         }
 103
 104         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 105         return ret;
 106 }
 107
 108 /*
 109  * Broadcast the event to the cpus, which are set in the mask
 110  */
 111 int tick_do_broadcast(cpumask_t mask)
 112 {
 113         int ret = 0, cpu = smp_processor_id();
 114         struct tick_device *td;
 115
 116         /*
 117          * Check, if the current cpu is in the mask
 118          */
 119         if (cpu_isset(cpu, mask)) {
 120                 cpu_clear(cpu, mask);
 121                 td = &per_cpu(tick_cpu_device, cpu);
 122                 td->evtdev->event_handler(td->evtdev);
 123                 ret = 1;
 124         }
 125
 126         if (!cpus_empty(mask)) {
 127                 /*
 128                  * It might be necessary to actually check whether the devices
 129                  * have different broadcast functions. For now, just use the
 130                  * one of the first device. This works as long as we have this
 131                  * misfeature only on x86 (lapic)
 132                  */
 133                 cpu = first_cpu(mask);
 134                 td = &per_cpu(tick_cpu_device, cpu);
 135                 td->evtdev->broadcast(mask);
 136                 ret = 1;
 137         }
 138         return ret;
 139 }
 140
 141 /*
 142  * Periodic broadcast:
 143  * - invoke the broadcast handlers
 144  */
 145 static void tick_do_periodic_broadcast(void)
 146 {
 147         cpumask_t mask;
 148
 149         spin_lock(&tick_broadcast_lock);
 150
 151         cpus_and(mask, cpu_online_map, tick_broadcast_mask);
 152         tick_do_broadcast(mask);
 153
 154         spin_unlock(&tick_broadcast_lock);
 155 }
 156
 157 /*
 158  * Event handler for periodic broadcast ticks
 159  */
 160 static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
 161 {
 162         dev->next_event.tv64 = KTIME_MAX;
 163
 164         tick_do_periodic_broadcast();
 165
 166         /*
 167          * The device is in periodic mode. No reprogramming necessary:
 168          */
 169         if (dev->mode == CLOCK_EVT_MODE_PERIODIC)
 170                 return;
 171
 172         /*
 173          * Setup the next period for devices, which do not have
 174          * periodic mode:
 175          */
 176         for (;;) {
 177                 ktime_t next = ktime_add(dev->next_event, tick_period);
 178
 179                 if (!clockevents_program_event(dev, next, ktime_get()))
 180                         return;
 181                 tick_do_periodic_broadcast();
 182         }
 183 }
 184
 185 /*
 186  * Powerstate information: The system enters/leaves a state, where
 187  * affected devices might stop
 188  */
 189 static void tick_do_broadcast_on_off(void *why)
 190 {
 191         struct clock_event_device *bc, *dev;
 192         struct tick_device *td;
 193         unsigned long flags, *reason = why;
 194         int cpu;
 195
 196         spin_lock_irqsave(&tick_broadcast_lock, flags);
 197
 198         cpu = smp_processor_id();
 199         td = &per_cpu(tick_cpu_device, cpu);
 200         dev = td->evtdev;
 201         bc = tick_broadcast_device.evtdev;
 202
 203         /*
 204          * Is the device in broadcast mode forever or is it not
 205          * affected by the powerstate ?
 206          */
 207         if (!dev || !tick_device_is_functional(dev) ||
 208             !(dev->features & CLOCK_EVT_FEAT_C3STOP))
 209                 goto out;
 210
 211         if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_ON) {
 212                 if (!cpu_isset(cpu, tick_broadcast_mask)) {
 213                         cpu_set(cpu, tick_broadcast_mask);
 214                         if (td->mode == TICKDEV_MODE_PERIODIC)
 215                                 clockevents_set_mode(dev,
 216                                                      CLOCK_EVT_MODE_SHUTDOWN);
 217                 }
 218         } else {
 219                 if (cpu_isset(cpu, tick_broadcast_mask)) {
 220                         cpu_clear(cpu, tick_broadcast_mask);
 221                         if (td->mode == TICKDEV_MODE_PERIODIC)
 222                                 tick_setup_periodic(dev, 0);
 223                 }
 224         }
 225
 226         if (cpus_empty(tick_broadcast_mask))
 227                 clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN);
 228         else {
 229                 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
 230                         tick_broadcast_start_periodic(bc);
 231                 else
 232                         tick_broadcast_setup_oneshot(bc);
 233         }
 234 out:
 235         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 236 }
 237
 238 /*
 239  * Powerstate information: The system enters/leaves a state, where
 240  * affected devices might stop.
 241  */
 242 void tick_broadcast_on_off(unsigned long reason, int *oncpu)
 243 {
 244         int cpu = get_cpu();
 245
 246         if (cpu == *oncpu)
 247                 tick_do_broadcast_on_off(&reason);
 248         else
 249                 smp_call_function_single(*oncpu, tick_do_broadcast_on_off,
 250                                          &reason, 1, 1);
 251         put_cpu();
 252 }
 253
 254 /*
 255  * Set the periodic handler depending on broadcast on/off
 256  */
 257 void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast)
 258 {
 259         if (!broadcast)
 260                 dev->event_handler = tick_handle_periodic;
 261         else
 262                 dev->event_handler = tick_handle_periodic_broadcast;
 263 }
 264
 265 /*
 266  * Remove a CPU from broadcasting
 267  */
 268 void tick_shutdown_broadcast(unsigned int *cpup)
 269 {
 270         struct clock_event_device *bc;
 271         unsigned long flags;
 272         unsigned int cpu = *cpup;
 273
 274         spin_lock_irqsave(&tick_broadcast_lock, flags);
 275
 276         bc = tick_broadcast_device.evtdev;
 277         cpu_clear(cpu, tick_broadcast_mask);
 278
 279         if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
 280                 if (bc && cpus_empty(tick_broadcast_mask))
 281                         clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN);
 282         }
 283
 284         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 285 }
 286
 287 #ifdef CONFIG_TICK_ONESHOT
 288
 289 static cpumask_t tick_broadcast_oneshot_mask;
 290
 291 /*
 292  * Debugging: see timer_list.c
 293  */
 294 cpumask_t *tick_get_broadcast_oneshot_mask(void)
 295 {
 296         return &tick_broadcast_oneshot_mask;
 297 }
 298
 299 static int tick_broadcast_set_event(ktime_t expires, int force)
 300 {
 301         struct clock_event_device *bc = tick_broadcast_device.evtdev;
 302         ktime_t now = ktime_get();
 303         int res;
 304
 305         for(;;) {
 306                 res = clockevents_program_event(bc, expires, now);
 307                 if (!res || !force)
 308                         return res;
 309                 now = ktime_get();
 310                 expires = ktime_add(now, ktime_set(0, bc->min_delta_ns));
 311         }
 312 }
 313
 314 /*
 315  * Reprogram the broadcast device:
 316  *
 317  * Called with tick_broadcast_lock held and interrupts disabled.
 318  */
 319 static int tick_broadcast_reprogram(void)
 320 {
 321         ktime_t expires = { .tv64 = KTIME_MAX };
 322         struct tick_device *td;
 323         int cpu;
 324
 325         /*
 326          * Find the event which expires next:
 327          */
 328         for (cpu = first_cpu(tick_broadcast_oneshot_mask); cpu != NR_CPUS;
 329              cpu = next_cpu(cpu, tick_broadcast_oneshot_mask)) {
 330                 td = &per_cpu(tick_cpu_device, cpu);
 331                 if (td->evtdev->next_event.tv64 < expires.tv64)
 332                         expires = td->evtdev->next_event;
 333         }
 334
 335         if (expires.tv64 == KTIME_MAX)
 336                 return 0;
 337
 338         return tick_broadcast_set_event(expires, 0);
 339 }
 340
 341 /*
 342  * Handle oneshot mode broadcasting
 343  */
 344 static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
 345 {
 346         struct tick_device *td;
 347         cpumask_t mask;
 348         ktime_t now;
 349         int cpu;
 350
 351         spin_lock(&tick_broadcast_lock);
 352 again:
 353         dev->next_event.tv64 = KTIME_MAX;
 354         mask = CPU_MASK_NONE;
 355         now = ktime_get();
 356         /* Find all expired events */
 357         for (cpu = first_cpu(tick_broadcast_oneshot_mask); cpu != NR_CPUS;
 358              cpu = next_cpu(cpu, tick_broadcast_oneshot_mask)) {
 359                 td = &per_cpu(tick_cpu_device, cpu);
 360                 if (td->evtdev->next_event.tv64 <= now.tv64)
 361                         cpu_set(cpu, mask);
 362         }
 363
 364         /*
 365          * Wakeup the cpus which have an expired event. The broadcast
 366          * device is reprogrammed in the return from idle code.
 367          */
 368         if (!tick_do_broadcast(mask)) {
 369                 /*
 370                  * The global event did not expire any CPU local
 371                  * events. This happens in dyntick mode, as the
 372                  * maximum PIT delta is quite small.
 373                  */
 374                 if (tick_broadcast_reprogram())
 375                         goto again;
 376         }
 377         spin_unlock(&tick_broadcast_lock);
 378 }
 379
 380 /*
 381  * Powerstate information: The system enters/leaves a state, where
 382  * affected devices might stop
 383  */
 384 void tick_broadcast_oneshot_control(unsigned long reason)
 385 {
 386         struct clock_event_device *bc, *dev;
 387         struct tick_device *td;
 388         unsigned long flags;
 389         int cpu;
 390
 391         spin_lock_irqsave(&tick_broadcast_lock, flags);
 392
 393         /*
 394          * Periodic mode does not care about the enter/exit of power
 395          * states
 396          */
 397         if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
 398                 goto out;
 399
 400         bc = tick_broadcast_device.evtdev;
 401         cpu = smp_processor_id();
 402         td = &per_cpu(tick_cpu_device, cpu);
 403         dev = td->evtdev;
 404
 405         if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
 406                 goto out;
 407
 408         if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) {
 409                 if (!cpu_isset(cpu, tick_broadcast_oneshot_mask)) {
 410                         cpu_set(cpu, tick_broadcast_oneshot_mask);
 411                         clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN);
 412                         if (dev->next_event.tv64 < bc->next_event.tv64)
 413                                 tick_broadcast_set_event(dev->next_event, 1);
 414                 }
 415         } else {
 416                 if (cpu_isset(cpu, tick_broadcast_oneshot_mask)) {
 417                         cpu_clear(cpu, tick_broadcast_oneshot_mask);
 418                         clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT);
 419                         if (dev->next_event.tv64 != KTIME_MAX)
 420                                 tick_program_event(dev->next_event, 1);
 421                 }
 422         }
 423
 424 out:
 425         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 426 }
 427
 428 /**
 429  * tick_broadcast_setup_highres - setup the broadcast device for highres
 430  */
 431 void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
 432 {
 433         if (bc->mode != CLOCK_EVT_MODE_ONESHOT) {
 434                 bc->event_handler = tick_handle_oneshot_broadcast;
 435                 clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
 436                 bc->next_event.tv64 = KTIME_MAX;
 437         }
 438 }
 439
 440 /*
 441  * Select oneshot operating mode for the broadcast device
 442  */
 443 void tick_broadcast_switch_to_oneshot(void)
 444 {
 445         struct clock_event_device *bc;
 446         unsigned long flags;
 447
 448         spin_lock_irqsave(&tick_broadcast_lock, flags);
 449
 450         tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
 451         bc = tick_broadcast_device.evtdev;
 452         if (bc)
 453                 tick_broadcast_setup_oneshot(bc);
 454         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 455 }
 456
 457
 458 /*
 459  * Remove a dead CPU from broadcasting
 460  */
 461 void tick_shutdown_broadcast_oneshot(unsigned int *cpup)
 462 {
 463         struct clock_event_device *bc;
 464         unsigned long flags;
 465         unsigned int cpu = *cpup;
 466
 467         spin_lock_irqsave(&tick_broadcast_lock, flags);
 468
 469         bc = tick_broadcast_device.evtdev;
 470         cpu_clear(cpu, tick_broadcast_oneshot_mask);
 471
 472         if (tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT) {
 473                 if (bc && cpus_empty(tick_broadcast_oneshot_mask))
 474                         clockevents_set_mode(bc, CLOCK_EVT_MODE_SHUTDOWN);
 475         }
 476
 477         spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 478 }
 479
 480 #endif