clockevents: check broadcast tick device not the clock events device
[pandora-kernel.git] / kernel / hrtimer.c
index 421be5f..cdec83e 100644 (file)
@@ -300,11 +300,10 @@ EXPORT_SYMBOL_GPL(ktime_sub_ns);
  */
 u64 ktime_divns(const ktime_t kt, s64 div)
 {
-       u64 dclc, inc, dns;
+       u64 dclc;
        int sft = 0;
 
-       dclc = dns = ktime_to_ns(kt);
-       inc = div;
+       dclc = ktime_to_ns(kt);
        /* Make sure the divisor is less than 2^32: */
        while (div >> 32) {
                sft++;
@@ -623,7 +622,7 @@ static void retrigger_next_event(void *arg)
 void clock_was_set(void)
 {
        /* Retrigger the CPU local events everywhere */
-       on_each_cpu(retrigger_next_event, NULL, 0, 1);
+       on_each_cpu(retrigger_next_event, NULL, 1);
 }
 
 /*
@@ -632,8 +631,6 @@ void clock_was_set(void)
  */
 void hres_timers_resume(void)
 {
-       WARN_ON_ONCE(num_online_cpus() > 1);
-
        /* Retrigger the CPU local events: */
        retrigger_next_event(NULL);
 }
@@ -675,13 +672,14 @@ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,
                         */
                        BUG_ON(timer->function(timer) != HRTIMER_NORESTART);
                        return 1;
-               case HRTIMER_CB_IRQSAFE_NO_SOFTIRQ:
+               case HRTIMER_CB_IRQSAFE_PERCPU:
+               case HRTIMER_CB_IRQSAFE_UNLOCKED:
                        /*
                         * This is solely for the sched tick emulation with
                         * dynamic tick support to ensure that we do not
                         * restart the tick right on the edge and end up with
                         * the tick timer in the softirq ! The calling site
-                        * takes care of this.
+                        * takes care of this. Also used for hrtimer sleeper !
                         */
                        debug_hrtimer_deactivate(timer);
                        return 1;
@@ -1003,10 +1001,18 @@ hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode)
         */
        raise = timer->state == HRTIMER_STATE_PENDING;
 
+       /*
+        * We use preempt_disable to prevent this task from migrating after
+        * setting up the softirq and raising it. Otherwise, if me migrate
+        * we will raise the softirq on the wrong CPU.
+        */
+       preempt_disable();
+
        unlock_hrtimer_base(timer, &flags);
 
        if (raise)
                hrtimer_raise_softirq();
+       preempt_enable();
 
        return ret;
 }
@@ -1078,7 +1084,7 @@ ktime_t hrtimer_get_remaining(const struct hrtimer *timer)
 }
 EXPORT_SYMBOL_GPL(hrtimer_get_remaining);
 
-#if defined(CONFIG_NO_IDLE_HZ) || defined(CONFIG_NO_HZ)
+#ifdef CONFIG_NO_HZ
 /**
  * hrtimer_get_next_event - get the time until next expiry event
  *
@@ -1240,7 +1246,8 @@ static void __run_hrtimer(struct hrtimer *timer)
        timer_stats_account_hrtimer(timer);
 
        fn = timer->function;
-       if (timer->cb_mode == HRTIMER_CB_IRQSAFE_NO_SOFTIRQ) {
+       if (timer->cb_mode == HRTIMER_CB_IRQSAFE_PERCPU ||
+           timer->cb_mode == HRTIMER_CB_IRQSAFE_UNLOCKED) {
                /*
                 * Used for scheduler timers, avoid lock inversion with
                 * rq->lock and tasklist_lock.
@@ -1447,7 +1454,7 @@ void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task)
        sl->timer.function = hrtimer_wakeup;
        sl->task = task;
 #ifdef CONFIG_HIGH_RES_TIMERS
-       sl->timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ;
+       sl->timer.cb_mode = HRTIMER_CB_IRQSAFE_UNLOCKED;
 #endif
 }
 
@@ -1586,29 +1593,95 @@ static void __cpuinit init_hrtimers_cpu(int cpu)
 
 #ifdef CONFIG_HOTPLUG_CPU
 
-static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
-                               struct hrtimer_clock_base *new_base)
+static int migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
+                               struct hrtimer_clock_base *new_base, int dcpu)
 {
        struct hrtimer *timer;
        struct rb_node *node;
+       int raise = 0;
 
        while ((node = rb_first(&old_base->active))) {
                timer = rb_entry(node, struct hrtimer, node);
                BUG_ON(hrtimer_callback_running(timer));
                debug_hrtimer_deactivate(timer);
-               __remove_hrtimer(timer, old_base, HRTIMER_STATE_INACTIVE, 0);
+
+               /*
+                * Should not happen. Per CPU timers should be
+                * canceled _before_ the migration code is called
+                */
+               if (timer->cb_mode == HRTIMER_CB_IRQSAFE_PERCPU) {
+                       __remove_hrtimer(timer, old_base,
+                                        HRTIMER_STATE_INACTIVE, 0);
+                       WARN(1, "hrtimer (%p %p)active but cpu %d dead\n",
+                            timer, timer->function, dcpu);
+                       continue;
+               }
+
+               /*
+                * Mark it as STATE_MIGRATE not INACTIVE otherwise the
+                * timer could be seen as !active and just vanish away
+                * under us on another CPU
+                */
+               __remove_hrtimer(timer, old_base, HRTIMER_STATE_MIGRATE, 0);
                timer->base = new_base;
                /*
                 * Enqueue the timer. Allow reprogramming of the event device
                 */
                enqueue_hrtimer(timer, new_base, 1);
+
+#ifdef CONFIG_HIGH_RES_TIMERS
+               /*
+                * Happens with high res enabled when the timer was
+                * already expired and the callback mode is
+                * HRTIMER_CB_IRQSAFE_UNLOCKED (hrtimer_sleeper). The
+                * enqueue code does not move them to the soft irq
+                * pending list for performance/latency reasons, but
+                * in the migration state, we need to do that
+                * otherwise we end up with a stale timer.
+                */
+               if (timer->state == HRTIMER_STATE_MIGRATE) {
+                       timer->state = HRTIMER_STATE_PENDING;
+                       list_add_tail(&timer->cb_entry,
+                                     &new_base->cpu_base->cb_pending);
+                       raise = 1;
+               }
+#endif
+               /* Clear the migration state bit */
+               timer->state &= ~HRTIMER_STATE_MIGRATE;
        }
+       return raise;
 }
 
+#ifdef CONFIG_HIGH_RES_TIMERS
+static int migrate_hrtimer_pending(struct hrtimer_cpu_base *old_base,
+                                  struct hrtimer_cpu_base *new_base)
+{
+       struct hrtimer *timer;
+       int raise = 0;
+
+       while (!list_empty(&old_base->cb_pending)) {
+               timer = list_entry(old_base->cb_pending.next,
+                                  struct hrtimer, cb_entry);
+
+               __remove_hrtimer(timer, timer->base, HRTIMER_STATE_PENDING, 0);
+               timer->base = &new_base->clock_base[timer->base->index];
+               list_add_tail(&timer->cb_entry, &new_base->cb_pending);
+               raise = 1;
+       }
+       return raise;
+}
+#else
+static int migrate_hrtimer_pending(struct hrtimer_cpu_base *old_base,
+                                  struct hrtimer_cpu_base *new_base)
+{
+       return 0;
+}
+#endif
+
 static void migrate_hrtimers(int cpu)
 {
        struct hrtimer_cpu_base *old_base, *new_base;
-       int i;
+       int i, raise = 0;
 
        BUG_ON(cpu_online(cpu));
        old_base = &per_cpu(hrtimer_bases, cpu);
@@ -1621,14 +1694,21 @@ static void migrate_hrtimers(int cpu)
        spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);
 
        for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
-               migrate_hrtimer_list(&old_base->clock_base[i],
-                                    &new_base->clock_base[i]);
+               if (migrate_hrtimer_list(&old_base->clock_base[i],
+                                        &new_base->clock_base[i], cpu))
+                       raise = 1;
        }
 
+       if (migrate_hrtimer_pending(old_base, new_base))
+               raise = 1;
+
        spin_unlock(&old_base->lock);
        spin_unlock(&new_base->lock);
        local_irq_enable();
        put_cpu_var(hrtimer_bases);
+
+       if (raise)
+               hrtimer_raise_softirq();
 }
 #endif /* CONFIG_HOTPLUG_CPU */
 
@@ -1669,7 +1749,7 @@ void __init hrtimers_init(void)
                          (void *)(long)smp_processor_id());
        register_cpu_notifier(&hrtimers_nb);
 #ifdef CONFIG_HIGH_RES_TIMERS
-       open_softirq(HRTIMER_SOFTIRQ, run_hrtimer_softirq, NULL);
+       open_softirq(HRTIMER_SOFTIRQ, run_hrtimer_softirq);
 #endif
 }