xen/blkback: don't include xen/evtchn.h
[pandora-kernel.git] / kernel / stop_machine.c
1 /*
2  * kernel/stop_machine.c
3  *
4  * Copyright (C) 2008, 2005     IBM Corporation.
5  * Copyright (C) 2008, 2005     Rusty Russell rusty@rustcorp.com.au
6  * Copyright (C) 2010           SUSE Linux Products GmbH
7  * Copyright (C) 2010           Tejun Heo <tj@kernel.org>
8  *
9  * This file is released under the GPLv2 and any later version.
10  */
11 #include <linux/completion.h>
12 #include <linux/cpu.h>
13 #include <linux/init.h>
14 #include <linux/kthread.h>
15 #include <linux/module.h>
16 #include <linux/percpu.h>
17 #include <linux/sched.h>
18 #include <linux/stop_machine.h>
19 #include <linux/interrupt.h>
20 #include <linux/kallsyms.h>
21
22 #include <asm/atomic.h>
23
24 /*
25  * Structure to determine completion condition and record errors.  May
26  * be shared by works on different cpus.
27  */
28 struct cpu_stop_done {
29         atomic_t                nr_todo;        /* nr left to execute */
30         bool                    executed;       /* actually executed? */
31         int                     ret;            /* collected return value */
32         struct completion       completion;     /* fired if nr_todo reaches 0 */
33 };
34
35 /* the actual stopper, one per every possible cpu, enabled on online cpus */
36 struct cpu_stopper {
37         spinlock_t              lock;
38         bool                    enabled;        /* is this stopper enabled? */
39         struct list_head        works;          /* list of pending works */
40         struct task_struct      *thread;        /* stopper thread */
41 };
42
43 static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper);
44
45 static void cpu_stop_init_done(struct cpu_stop_done *done, unsigned int nr_todo)
46 {
47         memset(done, 0, sizeof(*done));
48         atomic_set(&done->nr_todo, nr_todo);
49         init_completion(&done->completion);
50 }
51
52 /* signal completion unless @done is NULL */
53 static void cpu_stop_signal_done(struct cpu_stop_done *done, bool executed)
54 {
55         if (done) {
56                 if (executed)
57                         done->executed = true;
58                 if (atomic_dec_and_test(&done->nr_todo))
59                         complete(&done->completion);
60         }
61 }
62
63 /* queue @work to @stopper.  if offline, @work is completed immediately */
64 static void cpu_stop_queue_work(struct cpu_stopper *stopper,
65                                 struct cpu_stop_work *work)
66 {
67         unsigned long flags;
68
69         spin_lock_irqsave(&stopper->lock, flags);
70
71         if (stopper->enabled) {
72                 list_add_tail(&work->list, &stopper->works);
73                 wake_up_process(stopper->thread);
74         } else
75                 cpu_stop_signal_done(work->done, false);
76
77         spin_unlock_irqrestore(&stopper->lock, flags);
78 }
79
80 /**
81  * stop_one_cpu - stop a cpu
82  * @cpu: cpu to stop
83  * @fn: function to execute
84  * @arg: argument to @fn
85  *
86  * Execute @fn(@arg) on @cpu.  @fn is run in a process context with
87  * the highest priority preempting any task on the cpu and
88  * monopolizing it.  This function returns after the execution is
89  * complete.
90  *
91  * This function doesn't guarantee @cpu stays online till @fn
92  * completes.  If @cpu goes down in the middle, execution may happen
93  * partially or fully on different cpus.  @fn should either be ready
94  * for that or the caller should ensure that @cpu stays online until
95  * this function completes.
96  *
97  * CONTEXT:
98  * Might sleep.
99  *
100  * RETURNS:
101  * -ENOENT if @fn(@arg) was not executed because @cpu was offline;
102  * otherwise, the return value of @fn.
103  */
104 int stop_one_cpu(unsigned int cpu, cpu_stop_fn_t fn, void *arg)
105 {
106         struct cpu_stop_done done;
107         struct cpu_stop_work work = { .fn = fn, .arg = arg, .done = &done };
108
109         cpu_stop_init_done(&done, 1);
110         cpu_stop_queue_work(&per_cpu(cpu_stopper, cpu), &work);
111         wait_for_completion(&done.completion);
112         return done.executed ? done.ret : -ENOENT;
113 }
114
115 /**
116  * stop_one_cpu_nowait - stop a cpu but don't wait for completion
117  * @cpu: cpu to stop
118  * @fn: function to execute
119  * @arg: argument to @fn
120  *
121  * Similar to stop_one_cpu() but doesn't wait for completion.  The
122  * caller is responsible for ensuring @work_buf is currently unused
123  * and will remain untouched until stopper starts executing @fn.
124  *
125  * CONTEXT:
126  * Don't care.
127  */
128 void stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg,
129                         struct cpu_stop_work *work_buf)
130 {
131         *work_buf = (struct cpu_stop_work){ .fn = fn, .arg = arg, };
132         cpu_stop_queue_work(&per_cpu(cpu_stopper, cpu), work_buf);
133 }
134
135 /* static data for stop_cpus */
136 static DEFINE_MUTEX(stop_cpus_mutex);
137 static DEFINE_PER_CPU(struct cpu_stop_work, stop_cpus_work);
138
139 int __stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg)
140 {
141         struct cpu_stop_work *work;
142         struct cpu_stop_done done;
143         unsigned int cpu;
144
145         /* initialize works and done */
146         for_each_cpu(cpu, cpumask) {
147                 work = &per_cpu(stop_cpus_work, cpu);
148                 work->fn = fn;
149                 work->arg = arg;
150                 work->done = &done;
151         }
152         cpu_stop_init_done(&done, cpumask_weight(cpumask));
153
154         /*
155          * Disable preemption while queueing to avoid getting
156          * preempted by a stopper which might wait for other stoppers
157          * to enter @fn which can lead to deadlock.
158          */
159         preempt_disable();
160         for_each_cpu(cpu, cpumask)
161                 cpu_stop_queue_work(&per_cpu(cpu_stopper, cpu),
162                                     &per_cpu(stop_cpus_work, cpu));
163         preempt_enable();
164
165         wait_for_completion(&done.completion);
166         return done.executed ? done.ret : -ENOENT;
167 }
168
169 /**
170  * stop_cpus - stop multiple cpus
171  * @cpumask: cpus to stop
172  * @fn: function to execute
173  * @arg: argument to @fn
174  *
175  * Execute @fn(@arg) on online cpus in @cpumask.  On each target cpu,
176  * @fn is run in a process context with the highest priority
177  * preempting any task on the cpu and monopolizing it.  This function
178  * returns after all executions are complete.
179  *
180  * This function doesn't guarantee the cpus in @cpumask stay online
181  * till @fn completes.  If some cpus go down in the middle, execution
182  * on the cpu may happen partially or fully on different cpus.  @fn
183  * should either be ready for that or the caller should ensure that
184  * the cpus stay online until this function completes.
185  *
186  * All stop_cpus() calls are serialized making it safe for @fn to wait
187  * for all cpus to start executing it.
188  *
189  * CONTEXT:
190  * Might sleep.
191  *
192  * RETURNS:
193  * -ENOENT if @fn(@arg) was not executed at all because all cpus in
194  * @cpumask were offline; otherwise, 0 if all executions of @fn
195  * returned 0, any non zero return value if any returned non zero.
196  */
197 int stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg)
198 {
199         int ret;
200
201         /* static works are used, process one request at a time */
202         mutex_lock(&stop_cpus_mutex);
203         ret = __stop_cpus(cpumask, fn, arg);
204         mutex_unlock(&stop_cpus_mutex);
205         return ret;
206 }
207
208 /**
209  * try_stop_cpus - try to stop multiple cpus
210  * @cpumask: cpus to stop
211  * @fn: function to execute
212  * @arg: argument to @fn
213  *
214  * Identical to stop_cpus() except that it fails with -EAGAIN if
215  * someone else is already using the facility.
216  *
217  * CONTEXT:
218  * Might sleep.
219  *
220  * RETURNS:
221  * -EAGAIN if someone else is already stopping cpus, -ENOENT if
222  * @fn(@arg) was not executed at all because all cpus in @cpumask were
223  * offline; otherwise, 0 if all executions of @fn returned 0, any non
224  * zero return value if any returned non zero.
225  */
226 int try_stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg)
227 {
228         int ret;
229
230         /* static works are used, process one request at a time */
231         if (!mutex_trylock(&stop_cpus_mutex))
232                 return -EAGAIN;
233         ret = __stop_cpus(cpumask, fn, arg);
234         mutex_unlock(&stop_cpus_mutex);
235         return ret;
236 }
237
238 static int cpu_stopper_thread(void *data)
239 {
240         struct cpu_stopper *stopper = data;
241         struct cpu_stop_work *work;
242         int ret;
243
244 repeat:
245         set_current_state(TASK_INTERRUPTIBLE);  /* mb paired w/ kthread_stop */
246
247         if (kthread_should_stop()) {
248                 __set_current_state(TASK_RUNNING);
249                 return 0;
250         }
251
252         work = NULL;
253         spin_lock_irq(&stopper->lock);
254         if (!list_empty(&stopper->works)) {
255                 work = list_first_entry(&stopper->works,
256                                         struct cpu_stop_work, list);
257                 list_del_init(&work->list);
258         }
259         spin_unlock_irq(&stopper->lock);
260
261         if (work) {
262                 cpu_stop_fn_t fn = work->fn;
263                 void *arg = work->arg;
264                 struct cpu_stop_done *done = work->done;
265                 char ksym_buf[KSYM_NAME_LEN] __maybe_unused;
266
267                 __set_current_state(TASK_RUNNING);
268
269                 /* cpu stop callbacks are not allowed to sleep */
270                 preempt_disable();
271
272                 ret = fn(arg);
273                 if (ret)
274                         done->ret = ret;
275
276                 /* restore preemption and check it's still balanced */
277                 preempt_enable();
278                 WARN_ONCE(preempt_count(),
279                           "cpu_stop: %s(%p) leaked preempt count\n",
280                           kallsyms_lookup((unsigned long)fn, NULL, NULL, NULL,
281                                           ksym_buf), arg);
282
283                 cpu_stop_signal_done(done, true);
284         } else
285                 schedule();
286
287         goto repeat;
288 }
289
290 extern void sched_set_stop_task(int cpu, struct task_struct *stop);
291
292 /* manage stopper for a cpu, mostly lifted from sched migration thread mgmt */
293 static int __cpuinit cpu_stop_cpu_callback(struct notifier_block *nfb,
294                                            unsigned long action, void *hcpu)
295 {
296         unsigned int cpu = (unsigned long)hcpu;
297         struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
298         struct task_struct *p;
299
300         switch (action & ~CPU_TASKS_FROZEN) {
301         case CPU_UP_PREPARE:
302                 BUG_ON(stopper->thread || stopper->enabled ||
303                        !list_empty(&stopper->works));
304                 p = kthread_create_on_node(cpu_stopper_thread,
305                                            stopper,
306                                            cpu_to_node(cpu),
307                                            "migration/%d", cpu);
308                 if (IS_ERR(p))
309                         return notifier_from_errno(PTR_ERR(p));
310                 get_task_struct(p);
311                 kthread_bind(p, cpu);
312                 sched_set_stop_task(cpu, p);
313                 stopper->thread = p;
314                 break;
315
316         case CPU_ONLINE:
317                 /* strictly unnecessary, as first user will wake it */
318                 wake_up_process(stopper->thread);
319                 /* mark enabled */
320                 spin_lock_irq(&stopper->lock);
321                 stopper->enabled = true;
322                 spin_unlock_irq(&stopper->lock);
323                 break;
324
325 #ifdef CONFIG_HOTPLUG_CPU
326         case CPU_UP_CANCELED:
327         case CPU_POST_DEAD:
328         {
329                 struct cpu_stop_work *work;
330
331                 sched_set_stop_task(cpu, NULL);
332                 /* kill the stopper */
333                 kthread_stop(stopper->thread);
334                 /* drain remaining works */
335                 spin_lock_irq(&stopper->lock);
336                 list_for_each_entry(work, &stopper->works, list)
337                         cpu_stop_signal_done(work->done, false);
338                 stopper->enabled = false;
339                 spin_unlock_irq(&stopper->lock);
340                 /* release the stopper */
341                 put_task_struct(stopper->thread);
342                 stopper->thread = NULL;
343                 break;
344         }
345 #endif
346         }
347
348         return NOTIFY_OK;
349 }
350
351 /*
352  * Give it a higher priority so that cpu stopper is available to other
353  * cpu notifiers.  It currently shares the same priority as sched
354  * migration_notifier.
355  */
356 static struct notifier_block __cpuinitdata cpu_stop_cpu_notifier = {
357         .notifier_call  = cpu_stop_cpu_callback,
358         .priority       = 10,
359 };
360
361 static int __init cpu_stop_init(void)
362 {
363         void *bcpu = (void *)(long)smp_processor_id();
364         unsigned int cpu;
365         int err;
366
367         for_each_possible_cpu(cpu) {
368                 struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
369
370                 spin_lock_init(&stopper->lock);
371                 INIT_LIST_HEAD(&stopper->works);
372         }
373
374         /* start one for the boot cpu */
375         err = cpu_stop_cpu_callback(&cpu_stop_cpu_notifier, CPU_UP_PREPARE,
376                                     bcpu);
377         BUG_ON(err != NOTIFY_OK);
378         cpu_stop_cpu_callback(&cpu_stop_cpu_notifier, CPU_ONLINE, bcpu);
379         register_cpu_notifier(&cpu_stop_cpu_notifier);
380
381         return 0;
382 }
383 early_initcall(cpu_stop_init);
384
385 #ifdef CONFIG_STOP_MACHINE
386
387 /* This controls the threads on each CPU. */
388 enum stopmachine_state {
389         /* Dummy starting state for thread. */
390         STOPMACHINE_NONE,
391         /* Awaiting everyone to be scheduled. */
392         STOPMACHINE_PREPARE,
393         /* Disable interrupts. */
394         STOPMACHINE_DISABLE_IRQ,
395         /* Run the function */
396         STOPMACHINE_RUN,
397         /* Exit */
398         STOPMACHINE_EXIT,
399 };
400
401 struct stop_machine_data {
402         int                     (*fn)(void *);
403         void                    *data;
404         /* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
405         unsigned int            num_threads;
406         const struct cpumask    *active_cpus;
407
408         enum stopmachine_state  state;
409         atomic_t                thread_ack;
410 };
411
412 static void set_state(struct stop_machine_data *smdata,
413                       enum stopmachine_state newstate)
414 {
415         /* Reset ack counter. */
416         atomic_set(&smdata->thread_ack, smdata->num_threads);
417         smp_wmb();
418         smdata->state = newstate;
419 }
420
421 /* Last one to ack a state moves to the next state. */
422 static void ack_state(struct stop_machine_data *smdata)
423 {
424         if (atomic_dec_and_test(&smdata->thread_ack))
425                 set_state(smdata, smdata->state + 1);
426 }
427
428 /* This is the cpu_stop function which stops the CPU. */
429 static int stop_machine_cpu_stop(void *data)
430 {
431         struct stop_machine_data *smdata = data;
432         enum stopmachine_state curstate = STOPMACHINE_NONE;
433         int cpu = smp_processor_id(), err = 0;
434         bool is_active;
435
436         if (!smdata->active_cpus)
437                 is_active = cpu == cpumask_first(cpu_online_mask);
438         else
439                 is_active = cpumask_test_cpu(cpu, smdata->active_cpus);
440
441         /* Simple state machine */
442         do {
443                 /* Chill out and ensure we re-read stopmachine_state. */
444                 cpu_relax();
445                 if (smdata->state != curstate) {
446                         curstate = smdata->state;
447                         switch (curstate) {
448                         case STOPMACHINE_DISABLE_IRQ:
449                                 local_irq_disable();
450                                 hard_irq_disable();
451                                 break;
452                         case STOPMACHINE_RUN:
453                                 if (is_active)
454                                         err = smdata->fn(smdata->data);
455                                 break;
456                         default:
457                                 break;
458                         }
459                         ack_state(smdata);
460                 }
461         } while (curstate != STOPMACHINE_EXIT);
462
463         local_irq_enable();
464         return err;
465 }
466
467 int __stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
468 {
469         struct stop_machine_data smdata = { .fn = fn, .data = data,
470                                             .num_threads = num_online_cpus(),
471                                             .active_cpus = cpus };
472
473         /* Set the initial state and stop all online cpus. */
474         set_state(&smdata, STOPMACHINE_PREPARE);
475         return stop_cpus(cpu_online_mask, stop_machine_cpu_stop, &smdata);
476 }
477
478 int stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
479 {
480         int ret;
481
482         /* No CPUs can come up or down during this. */
483         get_online_cpus();
484         ret = __stop_machine(fn, data, cpus);
485         put_online_cpus();
486         return ret;
487 }
488 EXPORT_SYMBOL_GPL(stop_machine);
489
490 #endif  /* CONFIG_STOP_MACHINE */