kernel/rtmutex.c

   1 /*
   2  * RT-Mutexes: simple blocking mutual exclusion locks with PI support
   3  *
   4  * started by Ingo Molnar and Thomas Gleixner.
   5  *
   6  *  Copyright (C) 2004-2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
   7  *  Copyright (C) 2005-2006 Timesys Corp., Thomas Gleixner <tglx@timesys.com>
   8  *  Copyright (C) 2005 Kihon Technologies Inc., Steven Rostedt
   9  *  Copyright (C) 2006 Esben Nielsen
  10  *
  11  *  See Documentation/rt-mutex-design.txt for details.
  12  */
  13 #include <linux/spinlock.h>
  14 #include <linux/export.h>
  15 #include <linux/sched.h>
  16 #include <linux/timer.h>
  17
  18 #include "rtmutex_common.h"
  19
  20 /*
  21  * lock->owner state tracking:
  22  *
  23  * lock->owner holds the task_struct pointer of the owner. Bit 0
  24  * is used to keep track of the "lock has waiters" state.
  25  *
  26  * owner        bit0
  27  * NULL         0       lock is free (fast acquire possible)
  28  * NULL         1       lock is free and has waiters and the top waiter
  29  *                              is going to take the lock*
  30  * taskpointer  0       lock is held (fast release possible)
  31  * taskpointer  1       lock is held and has waiters**
  32  *
  33  * The fast atomic compare exchange based acquire and release is only
  34  * possible when bit 0 of lock->owner is 0.
  35  *
  36  * (*) It also can be a transitional state when grabbing the lock
  37  * with ->wait_lock is held. To prevent any fast path cmpxchg to the lock,
  38  * we need to set the bit0 before looking at the lock, and the owner may be
  39  * NULL in this small time, hence this can be a transitional state.
  40  *
  41  * (**) There is a small time when bit 0 is set but there are no
  42  * waiters. This can happen when grabbing the lock in the slow path.
  43  * To prevent a cmpxchg of the owner releasing the lock, we need to
  44  * set this bit before looking at the lock.
  45  */
  46
  47 static void
  48 rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner)
  49 {
  50         unsigned long val = (unsigned long)owner;
  51
  52         if (rt_mutex_has_waiters(lock))
  53                 val |= RT_MUTEX_HAS_WAITERS;
  54
  55         lock->owner = (struct task_struct *)val;
  56 }
  57
  58 static inline void clear_rt_mutex_waiters(struct rt_mutex *lock)
  59 {
  60         lock->owner = (struct task_struct *)
  61                         ((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);
  62 }
  63
  64 static void fixup_rt_mutex_waiters(struct rt_mutex *lock)
  65 {
  66         unsigned long owner, *p = (unsigned long *) &lock->owner;
  67
  68         if (rt_mutex_has_waiters(lock))
  69                 return;
  70
  71         /*
  72          * The rbtree has no waiters enqueued, now make sure that the
  73          * lock->owner still has the waiters bit set, otherwise the
  74          * following can happen:
  75          *
  76          * CPU 0        CPU 1           CPU2
  77          * l->owner=T1
  78          *              rt_mutex_lock(l)
  79          *              lock(l->lock)
  80          *              l->owner = T1 | HAS_WAITERS;
  81          *              enqueue(T2)
  82          *              boost()
  83          *                unlock(l->lock)
  84          *              block()
  85          *
  86          *                              rt_mutex_lock(l)
  87          *                              lock(l->lock)
  88          *                              l->owner = T1 | HAS_WAITERS;
  89          *                              enqueue(T3)
  90          *                              boost()
  91          *                                unlock(l->lock)
  92          *                              block()
  93          *              signal(->T2)    signal(->T3)
  94          *              lock(l->lock)
  95          *              dequeue(T2)
  96          *              deboost()
  97          *                unlock(l->lock)
  98          *                              lock(l->lock)
  99          *                              dequeue(T3)
 100          *                               ==> wait list is empty
 101          *                              deboost()
 102          *                               unlock(l->lock)
 103          *              lock(l->lock)
 104          *              fixup_rt_mutex_waiters()
 105          *                if (wait_list_empty(l) {
 106          *                  l->owner = owner
 107          *                  owner = l->owner & ~HAS_WAITERS;
 108          *                    ==> l->owner = T1
 109          *                }
 110          *                              lock(l->lock)
 111          * rt_mutex_unlock(l)           fixup_rt_mutex_waiters()
 112          *                                if (wait_list_empty(l) {
 113          *                                  owner = l->owner & ~HAS_WAITERS;
 114          * cmpxchg(l->owner, T1, NULL)
 115          *  ===> Success (l->owner = NULL)
 116          *
 117          *                                  l->owner = owner
 118          *                                    ==> l->owner = T1
 119          *                                }
 120          *
 121          * With the check for the waiter bit in place T3 on CPU2 will not
 122          * overwrite. All tasks fiddling with the waiters bit are
 123          * serialized by l->lock, so nothing else can modify the waiters
 124          * bit. If the bit is set then nothing can change l->owner either
 125          * so the simple RMW is safe. The cmpxchg() will simply fail if it
 126          * happens in the middle of the RMW because the waiters bit is
 127          * still set.
 128          */
 129         owner = ACCESS_ONCE(*p);
 130         if (owner & RT_MUTEX_HAS_WAITERS)
 131                 ACCESS_ONCE(*p) = owner & ~RT_MUTEX_HAS_WAITERS;
 132 }
 133
 134 /*
 135  * We can speed up the acquire/release, if the architecture
 136  * supports cmpxchg and if there's no debugging state to be set up
 137  */
 138 #if defined(__HAVE_ARCH_CMPXCHG) && !defined(CONFIG_DEBUG_RT_MUTEXES)
 139 # define rt_mutex_cmpxchg(l,c,n)        (cmpxchg(&l->owner, c, n) == c)
 140 static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
 141 {
 142         unsigned long owner, *p = (unsigned long *) &lock->owner;
 143
 144         do {
 145                 owner = *p;
 146         } while (cmpxchg(p, owner, owner | RT_MUTEX_HAS_WAITERS) != owner);
 147 }
 148
 149 /*
 150  * Safe fastpath aware unlock:
 151  * 1) Clear the waiters bit
 152  * 2) Drop lock->wait_lock
 153  * 3) Try to unlock the lock with cmpxchg
 154  */
 155 static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock)
 156         __releases(lock->wait_lock)
 157 {
 158         struct task_struct *owner = rt_mutex_owner(lock);
 159
 160         clear_rt_mutex_waiters(lock);
 161         raw_spin_unlock(&lock->wait_lock);
 162         /*
 163          * If a new waiter comes in between the unlock and the cmpxchg
 164          * we have two situations:
 165          *
 166          * unlock(wait_lock);
 167          *                                      lock(wait_lock);
 168          * cmpxchg(p, owner, 0) == owner
 169          *                                      mark_rt_mutex_waiters(lock);
 170          *                                      acquire(lock);
 171          * or:
 172          *
 173          * unlock(wait_lock);
 174          *                                      lock(wait_lock);
 175          *                                      mark_rt_mutex_waiters(lock);
 176          *
 177          * cmpxchg(p, owner, 0) != owner
 178          *                                      enqueue_waiter();
 179          *                                      unlock(wait_lock);
 180          * lock(wait_lock);
 181          * wake waiter();
 182          * unlock(wait_lock);
 183          *                                      lock(wait_lock);
 184          *                                      acquire(lock);
 185          */
 186         return rt_mutex_cmpxchg(lock, owner, NULL);
 187 }
 188
 189 #else
 190 # define rt_mutex_cmpxchg(l,c,n)        (0)
 191 static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
 192 {
 193         lock->owner = (struct task_struct *)
 194                         ((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);
 195 }
 196
 197 /*
 198  * Simple slow path only version: lock->owner is protected by lock->wait_lock.
 199  */
 200 static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock)
 201         __releases(lock->wait_lock)
 202 {
 203         lock->owner = NULL;
 204         raw_spin_unlock(&lock->wait_lock);
 205         return true;
 206 }
 207 #endif
 208
 209 /*
 210  * Calculate task priority from the waiter list priority
 211  *
 212  * Return task->normal_prio when the waiter list is empty or when
 213  * the waiter is not allowed to do priority boosting
 214  */
 215 int rt_mutex_getprio(struct task_struct *task)
 216 {
 217         if (likely(!task_has_pi_waiters(task)))
 218                 return task->normal_prio;
 219
 220         return min(task_top_pi_waiter(task)->pi_list_entry.prio,
 221                    task->normal_prio);
 222 }
 223
 224 /*
 225  * Adjust the priority of a task, after its pi_waiters got modified.
 226  *
 227  * This can be both boosting and unboosting. task->pi_lock must be held.
 228  */
 229 static void __rt_mutex_adjust_prio(struct task_struct *task)
 230 {
 231         int prio = rt_mutex_getprio(task);
 232
 233         if (task->prio != prio)
 234                 rt_mutex_setprio(task, prio);
 235 }
 236
 237 /*
 238  * Adjust task priority (undo boosting). Called from the exit path of
 239  * rt_mutex_slowunlock() and rt_mutex_slowlock().
 240  *
 241  * (Note: We do this outside of the protection of lock->wait_lock to
 242  * allow the lock to be taken while or before we readjust the priority
 243  * of task. We do not use the spin_xx_mutex() variants here as we are
 244  * outside of the debug path.)
 245  */
 246 static void rt_mutex_adjust_prio(struct task_struct *task)
 247 {
 248         unsigned long flags;
 249
 250         raw_spin_lock_irqsave(&task->pi_lock, flags);
 251         __rt_mutex_adjust_prio(task);
 252         raw_spin_unlock_irqrestore(&task->pi_lock, flags);
 253 }
 254
 255 /*
 256  * Max number of times we'll walk the boosting chain:
 257  */
 258 int max_lock_depth = 1024;
 259
 260 static inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p)
 261 {
 262         return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL;
 263 }
 264
 265 /*
 266  * Adjust the priority chain. Also used for deadlock detection.
 267  * Decreases task's usage by one - may thus free the task.
 268  *
 269  * @task:       the task owning the mutex (owner) for which a chain walk is
 270  *              probably needed
 271  * @deadlock_detect: do we have to carry out deadlock detection?
 272  * @orig_lock:  the mutex (can be NULL if we are walking the chain to recheck
 273  *              things for a task that has just got its priority adjusted, and
 274  *              is waiting on a mutex)
 275  * @next_lock:  the mutex on which the owner of @orig_lock was blocked before
 276  *              we dropped its pi_lock. Is never dereferenced, only used for
 277  *              comparison to detect lock chain changes.
 278  * @orig_waiter: rt_mutex_waiter struct for the task that has just donated
 279  *              its priority to the mutex owner (can be NULL in the case
 280  *              depicted above or if the top waiter is gone away and we are
 281  *              actually deboosting the owner)
 282  * @top_task:   the current top waiter
 283  *
 284  * Returns 0 or -EDEADLK.
 285  */
 286 static int rt_mutex_adjust_prio_chain(struct task_struct *task,
 287                                       int deadlock_detect,
 288                                       struct rt_mutex *orig_lock,
 289                                       struct rt_mutex *next_lock,
 290                                       struct rt_mutex_waiter *orig_waiter,
 291                                       struct task_struct *top_task)
 292 {
 293         struct rt_mutex *lock;
 294         struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter;
 295         int detect_deadlock, ret = 0, depth = 0;
 296         unsigned long flags;
 297
 298         detect_deadlock = debug_rt_mutex_detect_deadlock(orig_waiter,
 299                                                          deadlock_detect);
 300
 301         /*
 302          * The (de)boosting is a step by step approach with a lot of
 303          * pitfalls. We want this to be preemptible and we want hold a
 304          * maximum of two locks per step. So we have to check
 305          * carefully whether things change under us.
 306          */
 307  again:
 308         if (++depth > max_lock_depth) {
 309                 static int prev_max;
 310
 311                 /*
 312                  * Print this only once. If the admin changes the limit,
 313                  * print a new message when reaching the limit again.
 314                  */
 315                 if (prev_max != max_lock_depth) {
 316                         prev_max = max_lock_depth;
 317                         printk(KERN_WARNING "Maximum lock depth %d reached "
 318                                "task: %s (%d)\n", max_lock_depth,
 319                                top_task->comm, task_pid_nr(top_task));
 320                 }
 321                 put_task_struct(task);
 322
 323                 return -EDEADLK;
 324         }
 325  retry:
 326         /*
 327          * Task can not go away as we did a get_task() before !
 328          */
 329         raw_spin_lock_irqsave(&task->pi_lock, flags);
 330
 331         waiter = task->pi_blocked_on;
 332         /*
 333          * Check whether the end of the boosting chain has been
 334          * reached or the state of the chain has changed while we
 335          * dropped the locks.
 336          */
 337         if (!waiter)
 338                 goto out_unlock_pi;
 339
 340         /*
 341          * Check the orig_waiter state. After we dropped the locks,
 342          * the previous owner of the lock might have released the lock.
 343          */
 344         if (orig_waiter && !rt_mutex_owner(orig_lock))
 345                 goto out_unlock_pi;
 346
 347         /*
 348          * We dropped all locks after taking a refcount on @task, so
 349          * the task might have moved on in the lock chain or even left
 350          * the chain completely and blocks now on an unrelated lock or
 351          * on @orig_lock.
 352          *
 353          * We stored the lock on which @task was blocked in @next_lock,
 354          * so we can detect the chain change.
 355          */
 356         if (next_lock != waiter->lock)
 357                 goto out_unlock_pi;
 358
 359         /*
 360          * Drop out, when the task has no waiters. Note,
 361          * top_waiter can be NULL, when we are in the deboosting
 362          * mode!
 363          */
 364         if (top_waiter) {
 365                 if (!task_has_pi_waiters(task))
 366                         goto out_unlock_pi;
 367                 /*
 368                  * If deadlock detection is off, we stop here if we
 369                  * are not the top pi waiter of the task.
 370                  */
 371                 if (!detect_deadlock && top_waiter != task_top_pi_waiter(task))
 372                         goto out_unlock_pi;
 373         }
 374
 375         /*
 376          * When deadlock detection is off then we check, if further
 377          * priority adjustment is necessary.
 378          */
 379         if (!detect_deadlock && waiter->list_entry.prio == task->prio)
 380                 goto out_unlock_pi;
 381
 382         lock = waiter->lock;
 383         if (!raw_spin_trylock(&lock->wait_lock)) {
 384                 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
 385                 cpu_relax();
 386                 goto retry;
 387         }
 388
 389         /*
 390          * Deadlock detection. If the lock is the same as the original
 391          * lock which caused us to walk the lock chain or if the
 392          * current lock is owned by the task which initiated the chain
 393          * walk, we detected a deadlock.
 394          */
 395         if (lock == orig_lock || rt_mutex_owner(lock) == top_task) {
 396                 debug_rt_mutex_deadlock(deadlock_detect, orig_waiter, lock);
 397                 raw_spin_unlock(&lock->wait_lock);
 398                 ret = -EDEADLK;
 399                 goto out_unlock_pi;
 400         }
 401
 402         top_waiter = rt_mutex_top_waiter(lock);
 403
 404         /* Requeue the waiter */
 405         plist_del(&waiter->list_entry, &lock->wait_list);
 406         waiter->list_entry.prio = task->prio;
 407         plist_add(&waiter->list_entry, &lock->wait_list);
 408
 409         /* Release the task */
 410         raw_spin_unlock_irqrestore(&task->pi_lock, flags);
 411         if (!rt_mutex_owner(lock)) {
 412                 /*
 413                  * If the requeue above changed the top waiter, then we need
 414                  * to wake the new top waiter up to try to get the lock.
 415                  */
 416
 417                 if (top_waiter != rt_mutex_top_waiter(lock))
 418                         wake_up_process(rt_mutex_top_waiter(lock)->task);
 419                 raw_spin_unlock(&lock->wait_lock);
 420                 goto out_put_task;
 421         }
 422         put_task_struct(task);
 423
 424         /* Grab the next task */
 425         task = rt_mutex_owner(lock);
 426         get_task_struct(task);
 427         raw_spin_lock_irqsave(&task->pi_lock, flags);
 428
 429         if (waiter == rt_mutex_top_waiter(lock)) {
 430                 /* Boost the owner */
 431                 plist_del(&top_waiter->pi_list_entry, &task->pi_waiters);
 432                 waiter->pi_list_entry.prio = waiter->list_entry.prio;
 433                 plist_add(&waiter->pi_list_entry, &task->pi_waiters);
 434                 __rt_mutex_adjust_prio(task);
 435
 436         } else if (top_waiter == waiter) {
 437                 /* Deboost the owner */
 438                 plist_del(&waiter->pi_list_entry, &task->pi_waiters);
 439                 waiter = rt_mutex_top_waiter(lock);
 440                 waiter->pi_list_entry.prio = waiter->list_entry.prio;
 441                 plist_add(&waiter->pi_list_entry, &task->pi_waiters);
 442                 __rt_mutex_adjust_prio(task);
 443         }
 444
 445         /*
 446          * Check whether the task which owns the current lock is pi
 447          * blocked itself. If yes we store a pointer to the lock for
 448          * the lock chain change detection above. After we dropped
 449          * task->pi_lock next_lock cannot be dereferenced anymore.
 450          */
 451         next_lock = task_blocked_on_lock(task);
 452
 453         raw_spin_unlock_irqrestore(&task->pi_lock, flags);
 454
 455         top_waiter = rt_mutex_top_waiter(lock);
 456         raw_spin_unlock(&lock->wait_lock);
 457
 458         /*
 459          * We reached the end of the lock chain. Stop right here. No
 460          * point to go back just to figure that out.
 461          */
 462         if (!next_lock)
 463                 goto out_put_task;
 464
 465         if (!detect_deadlock && waiter != top_waiter)
 466                 goto out_put_task;
 467
 468         goto again;
 469
 470  out_unlock_pi:
 471         raw_spin_unlock_irqrestore(&task->pi_lock, flags);
 472  out_put_task:
 473         put_task_struct(task);
 474
 475         return ret;
 476 }
 477
 478 /*
 479  * Try to take an rt-mutex
 480  *
 481  * Must be called with lock->wait_lock held.
 482  *
 483  * @lock:   the lock to be acquired.
 484  * @task:   the task which wants to acquire the lock
 485  * @waiter: the waiter that is queued to the lock's wait list. (could be NULL)
 486  */
 487 static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
 488                 struct rt_mutex_waiter *waiter)
 489 {
 490         /*
 491          * We have to be careful here if the atomic speedups are
 492          * enabled, such that, when
 493          *  - no other waiter is on the lock
 494          *  - the lock has been released since we did the cmpxchg
 495          * the lock can be released or taken while we are doing the
 496          * checks and marking the lock with RT_MUTEX_HAS_WAITERS.
 497          *
 498          * The atomic acquire/release aware variant of
 499          * mark_rt_mutex_waiters uses a cmpxchg loop. After setting
 500          * the WAITERS bit, the atomic release / acquire can not
 501          * happen anymore and lock->wait_lock protects us from the
 502          * non-atomic case.
 503          *
 504          * Note, that this might set lock->owner =
 505          * RT_MUTEX_HAS_WAITERS in the case the lock is not contended
 506          * any more. This is fixed up when we take the ownership.
 507          * This is the transitional state explained at the top of this file.
 508          */
 509         mark_rt_mutex_waiters(lock);
 510
 511         if (rt_mutex_owner(lock))
 512                 return 0;
 513
 514         /*
 515          * It will get the lock because of one of these conditions:
 516          * 1) there is no waiter
 517          * 2) higher priority than waiters
 518          * 3) it is top waiter
 519          */
 520         if (rt_mutex_has_waiters(lock)) {
 521                 if (task->prio >= rt_mutex_top_waiter(lock)->list_entry.prio) {
 522                         if (!waiter || waiter != rt_mutex_top_waiter(lock))
 523                                 return 0;
 524                 }
 525         }
 526
 527         if (waiter || rt_mutex_has_waiters(lock)) {
 528                 unsigned long flags;
 529                 struct rt_mutex_waiter *top;
 530
 531                 raw_spin_lock_irqsave(&task->pi_lock, flags);
 532
 533                 /* remove the queued waiter. */
 534                 if (waiter) {
 535                         plist_del(&waiter->list_entry, &lock->wait_list);
 536                         task->pi_blocked_on = NULL;
 537                 }
 538
 539                 /*
 540                  * We have to enqueue the top waiter(if it exists) into
 541                  * task->pi_waiters list.
 542                  */
 543                 if (rt_mutex_has_waiters(lock)) {
 544                         top = rt_mutex_top_waiter(lock);
 545                         top->pi_list_entry.prio = top->list_entry.prio;
 546                         plist_add(&top->pi_list_entry, &task->pi_waiters);
 547                 }
 548                 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
 549         }
 550
 551         /* We got the lock. */
 552         debug_rt_mutex_lock(lock);
 553
 554         rt_mutex_set_owner(lock, task);
 555
 556         rt_mutex_deadlock_account_lock(lock, task);
 557
 558         return 1;
 559 }
 560
 561 /*
 562  * Task blocks on lock.
 563  *
 564  * Prepare waiter and propagate pi chain
 565  *
 566  * This must be called with lock->wait_lock held.
 567  */
 568 static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
 569                                    struct rt_mutex_waiter *waiter,
 570                                    struct task_struct *task,
 571                                    int detect_deadlock)
 572 {
 573         struct task_struct *owner = rt_mutex_owner(lock);
 574         struct rt_mutex_waiter *top_waiter = waiter;
 575         struct rt_mutex *next_lock;
 576         int chain_walk = 0, res;
 577         unsigned long flags;
 578
 579         /*
 580          * Early deadlock detection. We really don't want the task to
 581          * enqueue on itself just to untangle the mess later. It's not
 582          * only an optimization. We drop the locks, so another waiter
 583          * can come in before the chain walk detects the deadlock. So
 584          * the other will detect the deadlock and return -EDEADLOCK,
 585          * which is wrong, as the other waiter is not in a deadlock
 586          * situation.
 587          */
 588         if (owner == task)
 589                 return -EDEADLK;
 590
 591         raw_spin_lock_irqsave(&task->pi_lock, flags);
 592         __rt_mutex_adjust_prio(task);
 593         waiter->task = task;
 594         waiter->lock = lock;
 595         plist_node_init(&waiter->list_entry, task->prio);
 596         plist_node_init(&waiter->pi_list_entry, task->prio);
 597
 598         /* Get the top priority waiter on the lock */
 599         if (rt_mutex_has_waiters(lock))
 600                 top_waiter = rt_mutex_top_waiter(lock);
 601         plist_add(&waiter->list_entry, &lock->wait_list);
 602
 603         task->pi_blocked_on = waiter;
 604
 605         raw_spin_unlock_irqrestore(&task->pi_lock, flags);
 606
 607         if (!owner)
 608                 return 0;
 609
 610         raw_spin_lock_irqsave(&owner->pi_lock, flags);
 611         if (waiter == rt_mutex_top_waiter(lock)) {
 612                 plist_del(&top_waiter->pi_list_entry, &owner->pi_waiters);
 613                 plist_add(&waiter->pi_list_entry, &owner->pi_waiters);
 614
 615                 __rt_mutex_adjust_prio(owner);
 616                 if (owner->pi_blocked_on)
 617                         chain_walk = 1;
 618         } else if (debug_rt_mutex_detect_deadlock(waiter, detect_deadlock)) {
 619                 chain_walk = 1;
 620         }
 621
 622         /* Store the lock on which owner is blocked or NULL */
 623         next_lock = task_blocked_on_lock(owner);
 624
 625         raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
 626         /*
 627          * Even if full deadlock detection is on, if the owner is not
 628          * blocked itself, we can avoid finding this out in the chain
 629          * walk.
 630          */
 631         if (!chain_walk || !next_lock)
 632                 return 0;
 633
 634         /*
 635          * The owner can't disappear while holding a lock,
 636          * so the owner struct is protected by wait_lock.
 637          * Gets dropped in rt_mutex_adjust_prio_chain()!
 638          */
 639         get_task_struct(owner);
 640
 641         raw_spin_unlock(&lock->wait_lock);
 642
 643         res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock,
 644                                          next_lock, waiter, task);
 645
 646         raw_spin_lock(&lock->wait_lock);
 647
 648         return res;
 649 }
 650
 651 /*
 652  * Wake up the next waiter on the lock.
 653  *
 654  * Remove the top waiter from the current tasks pi waiter list and
 655  * wake it up.
 656  *
 657  * Called with lock->wait_lock held.
 658  */
 659 static void wakeup_next_waiter(struct rt_mutex *lock)
 660 {
 661         struct rt_mutex_waiter *waiter;
 662         unsigned long flags;
 663
 664         raw_spin_lock_irqsave(&current->pi_lock, flags);
 665
 666         waiter = rt_mutex_top_waiter(lock);
 667
 668         /*
 669          * Remove it from current->pi_waiters. We do not adjust a
 670          * possible priority boost right now. We execute wakeup in the
 671          * boosted mode and go back to normal after releasing
 672          * lock->wait_lock.
 673          */
 674         plist_del(&waiter->pi_list_entry, &current->pi_waiters);
 675
 676         /*
 677          * As we are waking up the top waiter, and the waiter stays
 678          * queued on the lock until it gets the lock, this lock
 679          * obviously has waiters. Just set the bit here and this has
 680          * the added benefit of forcing all new tasks into the
 681          * slow path making sure no task of lower priority than
 682          * the top waiter can steal this lock.
 683          */
 684         lock->owner = (void *) RT_MUTEX_HAS_WAITERS;
 685
 686         raw_spin_unlock_irqrestore(&current->pi_lock, flags);
 687
 688         /*
 689          * It's safe to dereference waiter as it cannot go away as
 690          * long as we hold lock->wait_lock. The waiter task needs to
 691          * acquire it in order to dequeue the waiter.
 692          */
 693         wake_up_process(waiter->task);
 694 }
 695
 696 /*
 697  * Remove a waiter from a lock and give up
 698  *
 699  * Must be called with lock->wait_lock held and
 700  * have just failed to try_to_take_rt_mutex().
 701  */
 702 static void remove_waiter(struct rt_mutex *lock,
 703                           struct rt_mutex_waiter *waiter)
 704 {
 705         int first = (waiter == rt_mutex_top_waiter(lock));
 706         struct task_struct *owner = rt_mutex_owner(lock);
 707         struct rt_mutex *next_lock = NULL;
 708         unsigned long flags;
 709
 710         raw_spin_lock_irqsave(&current->pi_lock, flags);
 711         plist_del(&waiter->list_entry, &lock->wait_list);
 712         current->pi_blocked_on = NULL;
 713         raw_spin_unlock_irqrestore(&current->pi_lock, flags);
 714
 715         if (!owner)
 716                 return;
 717
 718         if (first) {
 719
 720                 raw_spin_lock_irqsave(&owner->pi_lock, flags);
 721
 722                 plist_del(&waiter->pi_list_entry, &owner->pi_waiters);
 723
 724                 if (rt_mutex_has_waiters(lock)) {
 725                         struct rt_mutex_waiter *next;
 726
 727                         next = rt_mutex_top_waiter(lock);
 728                         plist_add(&next->pi_list_entry, &owner->pi_waiters);
 729                 }
 730                 __rt_mutex_adjust_prio(owner);
 731
 732                 /* Store the lock on which owner is blocked or NULL */
 733                 next_lock = task_blocked_on_lock(owner);
 734
 735                 raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
 736         }
 737
 738         WARN_ON(!plist_node_empty(&waiter->pi_list_entry));
 739
 740         if (!next_lock)
 741                 return;
 742
 743         /* gets dropped in rt_mutex_adjust_prio_chain()! */
 744         get_task_struct(owner);
 745
 746         raw_spin_unlock(&lock->wait_lock);
 747
 748         rt_mutex_adjust_prio_chain(owner, 0, lock, next_lock, NULL, current);
 749
 750         raw_spin_lock(&lock->wait_lock);
 751 }
 752
 753 /*
 754  * Recheck the pi chain, in case we got a priority setting
 755  *
 756  * Called from sched_setscheduler
 757  */
 758 void rt_mutex_adjust_pi(struct task_struct *task)
 759 {
 760         struct rt_mutex_waiter *waiter;
 761         struct rt_mutex *next_lock;
 762         unsigned long flags;
 763
 764         raw_spin_lock_irqsave(&task->pi_lock, flags);
 765
 766         waiter = task->pi_blocked_on;
 767         if (!waiter || waiter->list_entry.prio == task->prio) {
 768                 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
 769                 return;
 770         }
 771         next_lock = waiter->lock;
 772         raw_spin_unlock_irqrestore(&task->pi_lock, flags);
 773
 774         /* gets dropped in rt_mutex_adjust_prio_chain()! */
 775         get_task_struct(task);
 776
 777         rt_mutex_adjust_prio_chain(task, 0, NULL, next_lock, NULL, task);
 778 }
 779
 780 /**
 781  * __rt_mutex_slowlock() - Perform the wait-wake-try-to-take loop
 782  * @lock:                the rt_mutex to take
 783  * @state:               the state the task should block in (TASK_INTERRUPTIBLE
 784  *                       or TASK_UNINTERRUPTIBLE)
 785  * @timeout:             the pre-initialized and started timer, or NULL for none
 786  * @waiter:              the pre-initialized rt_mutex_waiter
 787  *
 788  * lock->wait_lock must be held by the caller.
 789  */
 790 static int __sched
 791 __rt_mutex_slowlock(struct rt_mutex *lock, int state,
 792                     struct hrtimer_sleeper *timeout,
 793                     struct rt_mutex_waiter *waiter)
 794 {
 795         int ret = 0;
 796         int was_disabled;
 797
 798         for (;;) {
 799                 /* Try to acquire the lock: */
 800                 if (try_to_take_rt_mutex(lock, current, waiter))
 801                         break;
 802
 803                 /*
 804                  * TASK_INTERRUPTIBLE checks for signals and
 805                  * timeout. Ignored otherwise.
 806                  */
 807                 if (unlikely(state == TASK_INTERRUPTIBLE)) {
 808                         /* Signal pending? */
 809                         if (signal_pending(current))
 810                                 ret = -EINTR;
 811                         if (timeout && !timeout->task)
 812                                 ret = -ETIMEDOUT;
 813                         if (ret)
 814                                 break;
 815                 }
 816
 817                 raw_spin_unlock(&lock->wait_lock);
 818
 819                 was_disabled = irqs_disabled();
 820                 if (was_disabled)
 821                         local_irq_enable();
 822
 823                 debug_rt_mutex_print_deadlock(waiter);
 824
 825                 schedule_rt_mutex(lock);
 826
 827                 if (was_disabled)
 828                         local_irq_disable();
 829
 830                 raw_spin_lock(&lock->wait_lock);
 831                 set_current_state(state);
 832         }
 833
 834         return ret;
 835 }
 836
 837 static void rt_mutex_handle_deadlock(int res, int detect_deadlock,
 838                                      struct rt_mutex_waiter *w)
 839 {
 840         /*
 841          * If the result is not -EDEADLOCK or the caller requested
 842          * deadlock detection, nothing to do here.
 843          */
 844         if (res != -EDEADLOCK || detect_deadlock)
 845                 return;
 846
 847         /*
 848          * Yell lowdly and stop the task right here.
 849          */
 850         rt_mutex_print_deadlock(w);
 851         while (1) {
 852                 set_current_state(TASK_INTERRUPTIBLE);
 853                 schedule();
 854         }
 855 }
 856
 857 /*
 858  * Slow path lock function:
 859  */
 860 static int __sched
 861 rt_mutex_slowlock(struct rt_mutex *lock, int state,
 862                   struct hrtimer_sleeper *timeout,
 863                   int detect_deadlock)
 864 {
 865         struct rt_mutex_waiter waiter;
 866         int ret = 0;
 867
 868         debug_rt_mutex_init_waiter(&waiter);
 869
 870         raw_spin_lock(&lock->wait_lock);
 871
 872         /* Try to acquire the lock again: */
 873         if (try_to_take_rt_mutex(lock, current, NULL)) {
 874                 raw_spin_unlock(&lock->wait_lock);
 875                 return 0;
 876         }
 877
 878         set_current_state(state);
 879
 880         /* Setup the timer, when timeout != NULL */
 881         if (unlikely(timeout)) {
 882                 hrtimer_start_expires(&timeout->timer, HRTIMER_MODE_ABS);
 883                 if (!hrtimer_active(&timeout->timer))
 884                         timeout->task = NULL;
 885         }
 886
 887         ret = task_blocks_on_rt_mutex(lock, &waiter, current, detect_deadlock);
 888
 889         if (likely(!ret))
 890                 ret = __rt_mutex_slowlock(lock, state, timeout, &waiter);
 891
 892         set_current_state(TASK_RUNNING);
 893
 894         if (unlikely(ret)) {
 895                 remove_waiter(lock, &waiter);
 896                 rt_mutex_handle_deadlock(ret, detect_deadlock, &waiter);
 897         }
 898
 899         /*
 900          * try_to_take_rt_mutex() sets the waiter bit
 901          * unconditionally. We might have to fix that up.
 902          */
 903         fixup_rt_mutex_waiters(lock);
 904
 905         raw_spin_unlock(&lock->wait_lock);
 906
 907         /* Remove pending timer: */
 908         if (unlikely(timeout))
 909                 hrtimer_cancel(&timeout->timer);
 910
 911         debug_rt_mutex_free_waiter(&waiter);
 912
 913         return ret;
 914 }
 915
 916 /*
 917  * Slow path try-lock function:
 918  */
 919 static inline int
 920 rt_mutex_slowtrylock(struct rt_mutex *lock)
 921 {
 922         int ret = 0;
 923
 924         raw_spin_lock(&lock->wait_lock);
 925
 926         if (likely(rt_mutex_owner(lock) != current)) {
 927
 928                 ret = try_to_take_rt_mutex(lock, current, NULL);
 929                 /*
 930                  * try_to_take_rt_mutex() sets the lock waiters
 931                  * bit unconditionally. Clean this up.
 932                  */
 933                 fixup_rt_mutex_waiters(lock);
 934         }
 935
 936         raw_spin_unlock(&lock->wait_lock);
 937
 938         return ret;
 939 }
 940
 941 /*
 942  * Slow path to release a rt-mutex:
 943  */
 944 static void __sched
 945 rt_mutex_slowunlock(struct rt_mutex *lock)
 946 {
 947         raw_spin_lock(&lock->wait_lock);
 948
 949         debug_rt_mutex_unlock(lock);
 950
 951         rt_mutex_deadlock_account_unlock(current);
 952
 953         /*
 954          * We must be careful here if the fast path is enabled. If we
 955          * have no waiters queued we cannot set owner to NULL here
 956          * because of:
 957          *
 958          * foo->lock->owner = NULL;
 959          *                      rtmutex_lock(foo->lock);   <- fast path
 960          *                      free = atomic_dec_and_test(foo->refcnt);
 961          *                      rtmutex_unlock(foo->lock); <- fast path
 962          *                      if (free)
 963          *                              kfree(foo);
 964          * raw_spin_unlock(foo->lock->wait_lock);
 965          *
 966          * So for the fastpath enabled kernel:
 967          *
 968          * Nothing can set the waiters bit as long as we hold
 969          * lock->wait_lock. So we do the following sequence:
 970          *
 971          *      owner = rt_mutex_owner(lock);
 972          *      clear_rt_mutex_waiters(lock);
 973          *      raw_spin_unlock(&lock->wait_lock);
 974          *      if (cmpxchg(&lock->owner, owner, 0) == owner)
 975          *              return;
 976          *      goto retry;
 977          *
 978          * The fastpath disabled variant is simple as all access to
 979          * lock->owner is serialized by lock->wait_lock:
 980          *
 981          *      lock->owner = NULL;
 982          *      raw_spin_unlock(&lock->wait_lock);
 983          */
 984         while (!rt_mutex_has_waiters(lock)) {
 985                 /* Drops lock->wait_lock ! */
 986                 if (unlock_rt_mutex_safe(lock) == true)
 987                         return;
 988                 /* Relock the rtmutex and try again */
 989                 raw_spin_lock(&lock->wait_lock);
 990         }
 991
 992         /*
 993          * The wakeup next waiter path does not suffer from the above
 994          * race. See the comments there.
 995          */
 996         wakeup_next_waiter(lock);
 997
 998         raw_spin_unlock(&lock->wait_lock);
 999
1000         /* Undo pi boosting if necessary: */
1001         rt_mutex_adjust_prio(current);
1002 }
1003
1004 /*
1005  * debug aware fast / slowpath lock,trylock,unlock
1006  *
1007  * The atomic acquire/release ops are compiled away, when either the
1008  * architecture does not support cmpxchg or when debugging is enabled.
1009  */
1010 static inline int
1011 rt_mutex_fastlock(struct rt_mutex *lock, int state,
1012                   int detect_deadlock,
1013                   int (*slowfn)(struct rt_mutex *lock, int state,
1014                                 struct hrtimer_sleeper *timeout,
1015                                 int detect_deadlock))
1016 {
1017         if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) {
1018                 rt_mutex_deadlock_account_lock(lock, current);
1019                 return 0;
1020         } else
1021                 return slowfn(lock, state, NULL, detect_deadlock);
1022 }
1023
1024 static inline int
1025 rt_mutex_timed_fastlock(struct rt_mutex *lock, int state,
1026                         struct hrtimer_sleeper *timeout, int detect_deadlock,
1027                         int (*slowfn)(struct rt_mutex *lock, int state,
1028                                       struct hrtimer_sleeper *timeout,
1029                                       int detect_deadlock))
1030 {
1031         if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) {
1032                 rt_mutex_deadlock_account_lock(lock, current);
1033                 return 0;
1034         } else
1035                 return slowfn(lock, state, timeout, detect_deadlock);
1036 }
1037
1038 static inline int
1039 rt_mutex_fasttrylock(struct rt_mutex *lock,
1040                      int (*slowfn)(struct rt_mutex *lock))
1041 {
1042         if (likely(rt_mutex_cmpxchg(lock, NULL, current))) {
1043                 rt_mutex_deadlock_account_lock(lock, current);
1044                 return 1;
1045         }
1046         return slowfn(lock);
1047 }
1048
1049 static inline void
1050 rt_mutex_fastunlock(struct rt_mutex *lock,
1051                     void (*slowfn)(struct rt_mutex *lock))
1052 {
1053         if (likely(rt_mutex_cmpxchg(lock, current, NULL)))
1054                 rt_mutex_deadlock_account_unlock(current);
1055         else
1056                 slowfn(lock);
1057 }
1058
1059 /**
1060  * rt_mutex_lock - lock a rt_mutex
1061  *
1062  * @lock: the rt_mutex to be locked
1063  */
1064 void __sched rt_mutex_lock(struct rt_mutex *lock)
1065 {
1066         might_sleep();
1067
1068         rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, 0, rt_mutex_slowlock);
1069 }
1070 EXPORT_SYMBOL_GPL(rt_mutex_lock);
1071
1072 /**
1073  * rt_mutex_lock_interruptible - lock a rt_mutex interruptible
1074  *
1075  * @lock:               the rt_mutex to be locked
1076  * @detect_deadlock:    deadlock detection on/off
1077  *
1078  * Returns:
1079  *  0           on success
1080  * -EINTR       when interrupted by a signal
1081  * -EDEADLK     when the lock would deadlock (when deadlock detection is on)
1082  */
1083 int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock,
1084                                                  int detect_deadlock)
1085 {
1086         might_sleep();
1087
1088         return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE,
1089                                  detect_deadlock, rt_mutex_slowlock);
1090 }
1091 EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible);
1092
1093 /**
1094  * rt_mutex_timed_lock - lock a rt_mutex interruptible
1095  *                      the timeout structure is provided
1096  *                      by the caller
1097  *
1098  * @lock:               the rt_mutex to be locked
1099  * @timeout:            timeout structure or NULL (no timeout)
1100  * @detect_deadlock:    deadlock detection on/off
1101  *
1102  * Returns:
1103  *  0           on success
1104  * -EINTR       when interrupted by a signal
1105  * -ETIMEDOUT   when the timeout expired
1106  * -EDEADLK     when the lock would deadlock (when deadlock detection is on)
1107  */
1108 int
1109 rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout,
1110                     int detect_deadlock)
1111 {
1112         might_sleep();
1113
1114         return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,
1115                                        detect_deadlock, rt_mutex_slowlock);
1116 }
1117 EXPORT_SYMBOL_GPL(rt_mutex_timed_lock);
1118
1119 /**
1120  * rt_mutex_trylock - try to lock a rt_mutex
1121  *
1122  * @lock:       the rt_mutex to be locked
1123  *
1124  * Returns 1 on success and 0 on contention
1125  */
1126 int __sched rt_mutex_trylock(struct rt_mutex *lock)
1127 {
1128         return rt_mutex_fasttrylock(lock, rt_mutex_slowtrylock);
1129 }
1130 EXPORT_SYMBOL_GPL(rt_mutex_trylock);
1131
1132 /**
1133  * rt_mutex_unlock - unlock a rt_mutex
1134  *
1135  * @lock: the rt_mutex to be unlocked
1136  */
1137 void __sched rt_mutex_unlock(struct rt_mutex *lock)
1138 {
1139         rt_mutex_fastunlock(lock, rt_mutex_slowunlock);
1140 }
1141 EXPORT_SYMBOL_GPL(rt_mutex_unlock);
1142
1143 /**
1144  * rt_mutex_destroy - mark a mutex unusable
1145  * @lock: the mutex to be destroyed
1146  *
1147  * This function marks the mutex uninitialized, and any subsequent
1148  * use of the mutex is forbidden. The mutex must not be locked when
1149  * this function is called.
1150  */
1151 void rt_mutex_destroy(struct rt_mutex *lock)
1152 {
1153         WARN_ON(rt_mutex_is_locked(lock));
1154 #ifdef CONFIG_DEBUG_RT_MUTEXES
1155         lock->magic = NULL;
1156 #endif
1157 }
1158
1159 EXPORT_SYMBOL_GPL(rt_mutex_destroy);
1160
1161 /**
1162  * __rt_mutex_init - initialize the rt lock
1163  *
1164  * @lock: the rt lock to be initialized
1165  *
1166  * Initialize the rt lock to unlocked state.
1167  *
1168  * Initializing of a locked rt lock is not allowed
1169  */
1170 void __rt_mutex_init(struct rt_mutex *lock, const char *name)
1171 {
1172         lock->owner = NULL;
1173         raw_spin_lock_init(&lock->wait_lock);
1174         plist_head_init(&lock->wait_list);
1175
1176         debug_rt_mutex_init(lock, name);
1177 }
1178 EXPORT_SYMBOL_GPL(__rt_mutex_init);
1179
1180 /**
1181  * rt_mutex_init_proxy_locked - initialize and lock a rt_mutex on behalf of a
1182  *                              proxy owner
1183  *
1184  * @lock:       the rt_mutex to be locked
1185  * @proxy_owner:the task to set as owner
1186  *
1187  * No locking. Caller has to do serializing itself
1188  * Special API call for PI-futex support
1189  */
1190 void rt_mutex_init_proxy_locked(struct rt_mutex *lock,
1191                                 struct task_struct *proxy_owner)
1192 {
1193         __rt_mutex_init(lock, NULL);
1194         debug_rt_mutex_proxy_lock(lock, proxy_owner);
1195         rt_mutex_set_owner(lock, proxy_owner);
1196         rt_mutex_deadlock_account_lock(lock, proxy_owner);
1197 }
1198
1199 /**
1200  * rt_mutex_proxy_unlock - release a lock on behalf of owner
1201  *
1202  * @lock:       the rt_mutex to be locked
1203  *
1204  * No locking. Caller has to do serializing itself
1205  * Special API call for PI-futex support
1206  */
1207 void rt_mutex_proxy_unlock(struct rt_mutex *lock,
1208                            struct task_struct *proxy_owner)
1209 {
1210         debug_rt_mutex_proxy_unlock(lock);
1211         rt_mutex_set_owner(lock, NULL);
1212         rt_mutex_deadlock_account_unlock(proxy_owner);
1213 }
1214
1215 /**
1216  * rt_mutex_start_proxy_lock() - Start lock acquisition for another task
1217  * @lock:               the rt_mutex to take
1218  * @waiter:             the pre-initialized rt_mutex_waiter
1219  * @task:               the task to prepare
1220  * @detect_deadlock:    perform deadlock detection (1) or not (0)
1221  *
1222  * Returns:
1223  *  0 - task blocked on lock
1224  *  1 - acquired the lock for task, caller should wake it up
1225  * <0 - error
1226  *
1227  * Special API call for FUTEX_REQUEUE_PI support.
1228  */
1229 int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
1230                               struct rt_mutex_waiter *waiter,
1231                               struct task_struct *task, int detect_deadlock)
1232 {
1233         int ret;
1234
1235         raw_spin_lock(&lock->wait_lock);
1236
1237         if (try_to_take_rt_mutex(lock, task, NULL)) {
1238                 raw_spin_unlock(&lock->wait_lock);
1239                 return 1;
1240         }
1241
1242         /* We enforce deadlock detection for futexes */
1243         ret = task_blocks_on_rt_mutex(lock, waiter, task, 1);
1244
1245         if (ret && !rt_mutex_owner(lock)) {
1246                 /*
1247                  * Reset the return value. We might have
1248                  * returned with -EDEADLK and the owner
1249                  * released the lock while we were walking the
1250                  * pi chain.  Let the waiter sort it out.
1251                  */
1252                 ret = 0;
1253         }
1254
1255         if (unlikely(ret))
1256                 remove_waiter(lock, waiter);
1257
1258         raw_spin_unlock(&lock->wait_lock);
1259
1260         debug_rt_mutex_print_deadlock(waiter);
1261
1262         return ret;
1263 }
1264
1265 /**
1266  * rt_mutex_next_owner - return the next owner of the lock
1267  *
1268  * @lock: the rt lock query
1269  *
1270  * Returns the next owner of the lock or NULL
1271  *
1272  * Caller has to serialize against other accessors to the lock
1273  * itself.
1274  *
1275  * Special API call for PI-futex support
1276  */
1277 struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock)
1278 {
1279         if (!rt_mutex_has_waiters(lock))
1280                 return NULL;
1281
1282         return rt_mutex_top_waiter(lock)->task;
1283 }
1284
1285 /**
1286  * rt_mutex_finish_proxy_lock() - Complete lock acquisition
1287  * @lock:               the rt_mutex we were woken on
1288  * @to:                 the timeout, null if none. hrtimer should already have
1289  *                      been started.
1290  * @waiter:             the pre-initialized rt_mutex_waiter
1291  * @detect_deadlock:    perform deadlock detection (1) or not (0)
1292  *
1293  * Complete the lock acquisition started our behalf by another thread.
1294  *
1295  * Returns:
1296  *  0 - success
1297  * <0 - error, one of -EINTR, -ETIMEDOUT, or -EDEADLK
1298  *
1299  * Special API call for PI-futex requeue support
1300  */
1301 int rt_mutex_finish_proxy_lock(struct rt_mutex *lock,
1302                                struct hrtimer_sleeper *to,
1303                                struct rt_mutex_waiter *waiter,
1304                                int detect_deadlock)
1305 {
1306         int ret;
1307
1308         raw_spin_lock(&lock->wait_lock);
1309
1310         set_current_state(TASK_INTERRUPTIBLE);
1311
1312         ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter);
1313
1314         set_current_state(TASK_RUNNING);
1315
1316         if (unlikely(ret))
1317                 remove_waiter(lock, waiter);
1318
1319         /*
1320          * try_to_take_rt_mutex() sets the waiter bit unconditionally. We might
1321          * have to fix that up.
1322          */
1323         fixup_rt_mutex_waiters(lock);
1324
1325         raw_spin_unlock(&lock->wait_lock);
1326
1327         return ret;
1328 }