1 /* Copyright 2008, 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation.
2 * GPL v2 and any later version.
6 #include <linux/kthread.h>
7 #include <linux/module.h>
8 #include <linux/sched.h>
9 #include <linux/stop_machine.h>
10 #include <linux/syscalls.h>
11 #include <linux/interrupt.h>
13 #include <asm/atomic.h>
14 #include <asm/uaccess.h>
16 /* This controls the threads on each CPU. */
17 enum stopmachine_state {
18 /* Dummy starting state for thread. */
20 /* Awaiting everyone to be scheduled. */
22 /* Disable interrupts. */
23 STOPMACHINE_DISABLE_IRQ,
24 /* Run the function */
29 static enum stopmachine_state state;
31 struct stop_machine_data {
37 /* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
38 static unsigned int num_threads;
39 static atomic_t thread_ack;
40 static DEFINE_MUTEX(lock);
41 /* setup_lock protects refcount, stop_machine_wq and stop_machine_work. */
42 static DEFINE_MUTEX(setup_lock);
43 /* Users of stop_machine. */
45 static struct workqueue_struct *stop_machine_wq;
46 static struct stop_machine_data active, idle;
47 static const struct cpumask *active_cpus;
48 static void __percpu *stop_machine_work;
50 static void set_state(enum stopmachine_state newstate)
52 /* Reset ack counter. */
53 atomic_set(&thread_ack, num_threads);
58 /* Last one to ack a state moves to the next state. */
59 static void ack_state(void)
61 if (atomic_dec_and_test(&thread_ack))
65 /* This is the actual function which stops the CPU. It runs
66 * in the context of a dedicated stopmachine workqueue. */
67 static void stop_cpu(struct work_struct *unused)
69 enum stopmachine_state curstate = STOPMACHINE_NONE;
70 struct stop_machine_data *smdata = &idle;
71 int cpu = smp_processor_id();
75 if (cpu == cpumask_first(cpu_online_mask))
78 if (cpumask_test_cpu(cpu, active_cpus))
81 /* Simple state machine */
83 /* Chill out and ensure we re-read stopmachine_state. */
85 if (state != curstate) {
88 case STOPMACHINE_DISABLE_IRQ:
93 /* On multiple CPUs only a single error code
94 * is needed to tell that something failed. */
95 err = smdata->fn(smdata->data);
104 } while (curstate != STOPMACHINE_EXIT);
109 /* Callback for CPUs which aren't supposed to do anything. */
110 static int chill(void *unused)
115 int stop_machine_create(void)
117 mutex_lock(&setup_lock);
120 stop_machine_wq = create_rt_workqueue("kstop");
121 if (!stop_machine_wq)
123 stop_machine_work = alloc_percpu(struct work_struct);
124 if (!stop_machine_work)
128 mutex_unlock(&setup_lock);
133 destroy_workqueue(stop_machine_wq);
134 mutex_unlock(&setup_lock);
137 EXPORT_SYMBOL_GPL(stop_machine_create);
139 void stop_machine_destroy(void)
141 mutex_lock(&setup_lock);
145 destroy_workqueue(stop_machine_wq);
146 free_percpu(stop_machine_work);
148 mutex_unlock(&setup_lock);
150 EXPORT_SYMBOL_GPL(stop_machine_destroy);
152 int __stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
154 struct work_struct *sm_work;
157 /* Set up initial state. */
159 num_threads = num_online_cpus();
167 set_state(STOPMACHINE_PREPARE);
169 /* Schedule the stop_cpu work on all cpus: hold this CPU so one
170 * doesn't hit this CPU until we're ready. */
172 for_each_online_cpu(i) {
173 sm_work = per_cpu_ptr(stop_machine_work, i);
174 INIT_WORK(sm_work, stop_cpu);
175 queue_work_on(i, stop_machine_wq, sm_work);
177 /* This will release the thread on our CPU. */
179 flush_workqueue(stop_machine_wq);
185 int stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
189 ret = stop_machine_create();
192 /* No CPUs can come up or down during this. */
194 ret = __stop_machine(fn, data, cpus);
196 stop_machine_destroy();
199 EXPORT_SYMBOL_GPL(stop_machine);