locking/rtmutex: Prevent dequeue vs. unlock race

[pandora-kernel.git] / kernel / rtmutex.c

diff --git a/kernel/rtmutex.c b/kernel/rtmutex.c
index 1928f3d..c24b2c5 100644 (file)
--- a/kernel/rtmutex.c
+++ b/kernel/rtmutex.c
@@ -63,8 +63,72 @@ static inline void clear_rt_mutex_waiters(struct rt_mutex *lock)
 
 static void fixup_rt_mutex_waiters(struct rt_mutex *lock)
 {
-       if (!rt_mutex_has_waiters(lock))
-               clear_rt_mutex_waiters(lock);
+       unsigned long owner, *p = (unsigned long *) &lock->owner;
+
+       if (rt_mutex_has_waiters(lock))
+               return;
+
+       /*
+        * The rbtree has no waiters enqueued, now make sure that the
+        * lock->owner still has the waiters bit set, otherwise the
+        * following can happen:
+        *
+        * CPU 0        CPU 1           CPU2
+        * l->owner=T1
+        *              rt_mutex_lock(l)
+        *              lock(l->lock)
+        *              l->owner = T1 | HAS_WAITERS;
+        *              enqueue(T2)
+        *              boost()
+        *                unlock(l->lock)
+        *              block()
+        *
+        *                              rt_mutex_lock(l)
+        *                              lock(l->lock)
+        *                              l->owner = T1 | HAS_WAITERS;
+        *                              enqueue(T3)
+        *                              boost()
+        *                                unlock(l->lock)
+        *                              block()
+        *              signal(->T2)    signal(->T3)
+        *              lock(l->lock)
+        *              dequeue(T2)
+        *              deboost()
+        *                unlock(l->lock)
+        *                              lock(l->lock)
+        *                              dequeue(T3)
+        *                               ==> wait list is empty
+        *                              deboost()
+        *                               unlock(l->lock)
+        *              lock(l->lock)
+        *              fixup_rt_mutex_waiters()
+        *                if (wait_list_empty(l) {
+        *                  l->owner = owner
+        *                  owner = l->owner & ~HAS_WAITERS;
+        *                    ==> l->owner = T1
+        *                }
+        *                              lock(l->lock)
+        * rt_mutex_unlock(l)           fixup_rt_mutex_waiters()
+        *                                if (wait_list_empty(l) {
+        *                                  owner = l->owner & ~HAS_WAITERS;
+        * cmpxchg(l->owner, T1, NULL)
+        *  ===> Success (l->owner = NULL)
+        *
+        *                                  l->owner = owner
+        *                                    ==> l->owner = T1
+        *                                }
+        *
+        * With the check for the waiter bit in place T3 on CPU2 will not
+        * overwrite. All tasks fiddling with the waiters bit are
+        * serialized by l->lock, so nothing else can modify the waiters
+        * bit. If the bit is set then nothing can change l->owner either
+        * so the simple RMW is safe. The cmpxchg() will simply fail if it
+        * happens in the middle of the RMW because the waiters bit is
+        * still set.
+        */
+       owner = ACCESS_ONCE(*p);
+       if (owner & RT_MUTEX_HAS_WAITERS)
+               ACCESS_ONCE(*p) = owner & ~RT_MUTEX_HAS_WAITERS;
 }
 
 /*