include/asm-arm/mutex.h

   1 /*
   2  * include/asm-arm/mutex.h
   3  *
   4  * ARM optimized mutex locking primitives
   5  *
   6  * Please look into asm-generic/mutex-xchg.h for a formal definition.
   7  */
   8 #ifndef _ASM_MUTEX_H
   9 #define _ASM_MUTEX_H
  10
  11 #if __LINUX_ARM_ARCH__ < 6
  12 /* On pre-ARMv6 hardware the swp based implementation is the most efficient. */
  13 # include <asm-generic/mutex-xchg.h>
  14 #else
  15
  16 /*
  17  * Attempting to lock a mutex on ARMv6+ can be done with a bastardized
  18  * atomic decrement (it is not a reliable atomic decrement but it satisfies
  19  * the defined semantics for our purpose, while being smaller and faster
  20  * than a real atomic decrement or atomic swap.  The idea is to attempt
  21  * decrementing the lock value only once.  If once decremented it isn't zero,
  22  * or if its store-back fails due to a dispute on the exclusive store, we
  23  * simply bail out immediately through the slow path where the lock will be
  24  * reattempted until it succeeds.
  25  */
  26 #define __mutex_fastpath_lock(count, fail_fn)                           \
  27 do {                                                                    \
  28         int __ex_flag, __res;                                           \
  29                                                                         \
  30         typecheck(atomic_t *, count);                                   \
  31         typecheck_fn(fastcall void (*)(atomic_t *), fail_fn);           \
  32                                                                         \
  33         __asm__ (                                                       \
  34                 "ldrex  %0, [%2]        \n"                             \
  35                 "sub    %0, %0, #1      \n"                             \
  36                 "strex  %1, %0, [%2]    \n"                             \
  37                                                                         \
  38                 : "=&r" (__res), "=&r" (__ex_flag)                      \
  39                 : "r" (&(count)->counter)                               \
  40                 : "cc","memory" );                                      \
  41                                                                         \
  42         if (unlikely(__res || __ex_flag))                               \
  43                 fail_fn(count);                                         \
  44 } while (0)
  45
  46 #define __mutex_fastpath_lock_retval(count, fail_fn)                    \
  47 ({                                                                      \
  48         int __ex_flag, __res;                                           \
  49                                                                         \
  50         typecheck(atomic_t *, count);                                   \
  51         typecheck_fn(fastcall int (*)(atomic_t *), fail_fn);            \
  52                                                                         \
  53         __asm__ (                                                       \
  54                 "ldrex  %0, [%2]        \n"                             \
  55                 "sub    %0, %0, #1      \n"                             \
  56                 "strex  %1, %0, [%2]    \n"                             \
  57                                                                         \
  58                 : "=&r" (__res), "=&r" (__ex_flag)                      \
  59                 : "r" (&(count)->counter)                               \
  60                 : "cc","memory" );                                      \
  61                                                                         \
  62         __res |= __ex_flag;                                             \
  63         if (unlikely(__res != 0))                                       \
  64                 __res = fail_fn(count);                                 \
  65         __res;                                                          \
  66 })
  67
  68 /*
  69  * Same trick is used for the unlock fast path. However the original value,
  70  * rather than the result, is used to test for success in order to have
  71  * better generated assembly.
  72  */
  73 #define __mutex_fastpath_unlock(count, fail_fn)                         \
  74 do {                                                                    \
  75         int __ex_flag, __res, __orig;                                   \
  76                                                                         \
  77         typecheck(atomic_t *, count);                                   \
  78         typecheck_fn(fastcall void (*)(atomic_t *), fail_fn);           \
  79                                                                         \
  80         __asm__ (                                                       \
  81                 "ldrex  %0, [%3]        \n"                             \
  82                 "add    %1, %0, #1      \n"                             \
  83                 "strex  %2, %1, [%3]    \n"                             \
  84                                                                         \
  85                 : "=&r" (__orig), "=&r" (__res), "=&r" (__ex_flag)      \
  86                 : "r" (&(count)->counter)                               \
  87                 : "cc","memory" );                                      \
  88                                                                         \
  89         if (unlikely(__orig || __ex_flag))                              \
  90                 fail_fn(count);                                         \
  91 } while (0)
  92
  93 /*
  94  * If the unlock was done on a contended lock, or if the unlock simply fails
  95  * then the mutex remains locked.
  96  */
  97 #define __mutex_slowpath_needs_to_unlock()      1
  98
  99 /*
 100  * For __mutex_fastpath_trylock we use another construct which could be
 101  * described as a "single value cmpxchg".
 102  *
 103  * This provides the needed trylock semantics like cmpxchg would, but it is
 104  * lighter and less generic than a true cmpxchg implementation.
 105  */
 106 static inline int
 107 __mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *))
 108 {
 109         int __ex_flag, __res, __orig;
 110
 111         __asm__ (
 112
 113                 "1: ldrex       %0, [%3]        \n"
 114                 "subs           %1, %0, #1      \n"
 115                 "strexeq        %2, %1, [%3]    \n"
 116                 "movlt          %0, #0          \n"
 117                 "cmpeq          %2, #0          \n"
 118                 "bgt            1b              \n"
 119
 120                 : "=&r" (__orig), "=&r" (__res), "=&r" (__ex_flag)
 121                 : "r" (&count->counter)
 122                 : "cc", "memory" );
 123
 124         return __orig;
 125 }
 126
 127 #endif
 128 #endif