sched: fix __set_task_cpu() SMP race

[pandora-kernel.git] / kernel / sched.c

diff --git a/kernel/sched.c b/kernel/sched.c
index b18f231..db1f71e 100644 (file)
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -52,7 +52,6 @@
 #include <linux/cpu.h>
 #include <linux/cpuset.h>
 #include <linux/percpu.h>
-#include <linux/cpu_acct.h>
 #include <linux/kthread.h>
 #include <linux/seq_file.h>
 #include <linux/sysctl.h>
@@ -217,15 +216,15 @@ static inline struct task_group *task_group(struct task_struct *p)
 }
 
 /* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
-static inline void set_task_cfs_rq(struct task_struct *p)
+static inline void set_task_cfs_rq(struct task_struct *p, unsigned int cpu)
 {
-       p->se.cfs_rq = task_group(p)->cfs_rq[task_cpu(p)];
-       p->se.parent = task_group(p)->se[task_cpu(p)];
+       p->se.cfs_rq = task_group(p)->cfs_rq[cpu];
+       p->se.parent = task_group(p)->se[cpu];
 }
 
 #else
 
-static inline void set_task_cfs_rq(struct task_struct *p) { }
+static inline void set_task_cfs_rq(struct task_struct *p, unsigned int cpu) { }
 
 #endif /* CONFIG_FAIR_GROUP_SCHED */
 
@@ -1023,10 +1022,16 @@ unsigned long weighted_cpuload(const int cpu)
 
 static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
 {
+       set_task_cfs_rq(p, cpu);
 #ifdef CONFIG_SMP
+       /*
+        * After ->cpu is set up to a new value, task_rq_lock(p, ...) can be
+        * successfuly executed on another CPU. We must ensure that updates of
+        * per-task data have been completed by this moment.
+        */
+       smp_wmb();
        task_thread_info(p)->cpu = cpu;
 #endif
-       set_task_cfs_rq(p);
 }
 
 #ifdef CONFIG_SMP
@@ -3338,13 +3343,9 @@ void account_user_time(struct task_struct *p, cputime_t cputime)
 {
        struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
        cputime64_t tmp;
-       struct rq *rq = this_rq();
 
        p->utime = cputime_add(p->utime, cputime);
 
-       if (p != rq->idle)
-               cpuacct_charge(p, cputime);
-
        /* Add user time to cpustat. */
        tmp = cputime_to_cputime64(cputime);
        if (TASK_NICE(p) > 0)
@@ -3395,10 +3396,8 @@ void account_system_time(struct task_struct *p, int hardirq_offset,
        struct rq *rq = this_rq();
        cputime64_t tmp;
 
-       if (p->flags & PF_VCPU) {
-               account_guest_time(p, cputime);
-               return;
-       }
+       if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0))
+               return account_guest_time(p, cputime);
 
        p->stime = cputime_add(p->stime, cputime);
 
@@ -3408,10 +3407,9 @@ void account_system_time(struct task_struct *p, int hardirq_offset,
                cpustat->irq = cputime64_add(cpustat->irq, tmp);
        else if (softirq_count())
                cpustat->softirq = cputime64_add(cpustat->softirq, tmp);
-       else if (p != rq->idle) {
+       else if (p != rq->idle)
                cpustat->system = cputime64_add(cpustat->system, tmp);
-               cpuacct_charge(p, cputime);
-       } else if (atomic_read(&rq->nr_iowait) > 0)
+       else if (atomic_read(&rq->nr_iowait) > 0)
                cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
        else
                cpustat->idle = cputime64_add(cpustat->idle, tmp);
@@ -3447,10 +3445,8 @@ void account_steal_time(struct task_struct *p, cputime_t steal)
                        cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
                else
                        cpustat->idle = cputime64_add(cpustat->idle, tmp);
-       } else {
+       } else
                cpustat->steal = cputime64_add(cpustat->steal, tmp);
-               cpuacct_charge(p, -tmp);
-       }
 }
 
 /*
@@ -7097,8 +7093,10 @@ void sched_move_task(struct task_struct *tsk)
 
        rq = task_rq_lock(tsk, &flags);
 
-       if (tsk->sched_class != &fair_sched_class)
+       if (tsk->sched_class != &fair_sched_class) {
+               set_task_cfs_rq(tsk, task_cpu(tsk));
                goto done;
+       }
 
        update_rq_clock(rq);
 
@@ -7111,7 +7109,7 @@ void sched_move_task(struct task_struct *tsk)
                        tsk->sched_class->put_prev_task(rq, tsk);
        }
 
-       set_task_cfs_rq(tsk);
+       set_task_cfs_rq(tsk, task_cpu(tsk));
 
        if (on_rq) {
                if (unlikely(running))