arch/x86/include/asm/cmpxchg_32.h

   1 #ifndef _ASM_X86_CMPXCHG_32_H
   2 #define _ASM_X86_CMPXCHG_32_H
   3
   4 #include <linux/bitops.h> /* for LOCK_PREFIX */
   5
   6 /*
   7  * Note: if you use set64_bit(), __cmpxchg64(), or their variants, you
   8  *       you need to test for the feature in boot_cpu_data.
   9  */
  10
  11 extern void __xchg_wrong_size(void);
  12
  13 /*
  14  * Note: no "lock" prefix even on SMP: xchg always implies lock anyway.
  15  * Since this is generally used to protect other memory information, we
  16  * use "asm volatile" and "memory" clobbers to prevent gcc from moving
  17  * information around.
  18  */
  19 #define __xchg(x, ptr, size)                                            \
  20 ({                                                                      \
  21         __typeof(*(ptr)) __x = (x);                                     \
  22         switch (size) {                                                 \
  23         case 1:                                                         \
  24         {                                                               \
  25                 volatile u8 *__ptr = (volatile u8 *)(ptr);              \
  26                 asm volatile("xchgb %0,%1"                              \
  27                              : "=q" (__x), "+m" (*__ptr)                \
  28                              : "0" (__x)                                \
  29                              : "memory");                               \
  30                 break;                                                  \
  31         }                                                               \
  32         case 2:                                                         \
  33         {                                                               \
  34                 volatile u16 *__ptr = (volatile u16 *)(ptr);            \
  35                 asm volatile("xchgw %0,%1"                              \
  36                              : "=r" (__x), "+m" (*__ptr)                \
  37                              : "0" (__x)                                \
  38                              : "memory");                               \
  39                 break;                                                  \
  40         }                                                               \
  41         case 4:                                                         \
  42         {                                                               \
  43                 volatile u32 *__ptr = (volatile u32 *)(ptr);            \
  44                 asm volatile("xchgl %0,%1"                              \
  45                              : "=r" (__x), "+m" (*__ptr)                \
  46                              : "0" (__x)                                \
  47                              : "memory");                               \
  48                 break;                                                  \
  49         }                                                               \
  50         default:                                                        \
  51                 __xchg_wrong_size();                                    \
  52         }                                                               \
  53         __x;                                                            \
  54 })
  55
  56 #define xchg(ptr, v)                                                    \
  57         __xchg((v), (ptr), sizeof(*ptr))
  58
  59 /*
  60  * CMPXCHG8B only writes to the target if we had the previous
  61  * value in registers, otherwise it acts as a read and gives us the
  62  * "new previous" value.  That is why there is a loop.  Preloading
  63  * EDX:EAX is a performance optimization: in the common case it means
  64  * we need only one locked operation.
  65  *
  66  * A SIMD/3DNOW!/MMX/FPU 64-bit store here would require at the very
  67  * least an FPU save and/or %cr0.ts manipulation.
  68  *
  69  * cmpxchg8b must be used with the lock prefix here to allow the
  70  * instruction to be executed atomically.  We need to have the reader
  71  * side to see the coherent 64bit value.
  72  */
  73 static inline void set_64bit(volatile u64 *ptr, u64 value)
  74 {
  75         u32 low  = value;
  76         u32 high = value >> 32;
  77         u64 prev = *ptr;
  78
  79         asm volatile("\n1:\t"
  80                      LOCK_PREFIX "cmpxchg8b %0\n\t"
  81                      "jnz 1b"
  82                      : "=m" (*ptr), "+A" (prev)
  83                      : "b" (low), "c" (high)
  84                      : "memory");
  85 }
  86
  87 extern void __cmpxchg_wrong_size(void);
  88
  89 /*
  90  * Atomic compare and exchange.  Compare OLD with MEM, if identical,
  91  * store NEW in MEM.  Return the initial value in MEM.  Success is
  92  * indicated by comparing RETURN with OLD.
  93  */
  94 #define __raw_cmpxchg(ptr, old, new, size, lock)                        \
  95 ({                                                                      \
  96         __typeof__(*(ptr)) __ret;                                       \
  97         __typeof__(*(ptr)) __old = (old);                               \
  98         __typeof__(*(ptr)) __new = (new);                               \
  99         switch (size) {                                                 \
 100         case 1:                                                         \
 101         {                                                               \
 102                 volatile u8 *__ptr = (volatile u8 *)(ptr);              \
 103                 asm volatile(lock "cmpxchgb %2,%1"                      \
 104                              : "=a" (__ret), "+m" (*__ptr)              \
 105                              : "q" (__new), "0" (__old)                 \
 106                              : "memory");                               \
 107                 break;                                                  \
 108         }                                                               \
 109         case 2:                                                         \
 110         {                                                               \
 111                 volatile u16 *__ptr = (volatile u16 *)(ptr);            \
 112                 asm volatile(lock "cmpxchgw %2,%1"                      \
 113                              : "=a" (__ret), "+m" (*__ptr)              \
 114                              : "r" (__new), "0" (__old)                 \
 115                              : "memory");                               \
 116                 break;                                                  \
 117         }                                                               \
 118         case 4:                                                         \
 119         {                                                               \
 120                 volatile u32 *__ptr = (volatile u32 *)(ptr);            \
 121                 asm volatile(lock "cmpxchgl %2,%1"                      \
 122                              : "=a" (__ret), "+m" (*__ptr)              \
 123                              : "r" (__new), "0" (__old)                 \
 124                              : "memory");                               \
 125                 break;                                                  \
 126         }                                                               \
 127         default:                                                        \
 128                 __cmpxchg_wrong_size();                                 \
 129         }                                                               \
 130         __ret;                                                          \
 131 })
 132
 133 #define __cmpxchg(ptr, old, new, size)                                  \
 134         __raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX)
 135
 136 #define __sync_cmpxchg(ptr, old, new, size)                             \
 137         __raw_cmpxchg((ptr), (old), (new), (size), "lock; ")
 138
 139 #define __cmpxchg_local(ptr, old, new, size)                            \
 140         __raw_cmpxchg((ptr), (old), (new), (size), "")
 141
 142 #ifdef CONFIG_X86_CMPXCHG
 143 #define __HAVE_ARCH_CMPXCHG 1
 144
 145 #define cmpxchg(ptr, old, new)                                          \
 146         __cmpxchg((ptr), (old), (new), sizeof(*ptr))
 147
 148 #define sync_cmpxchg(ptr, old, new)                                     \
 149         __sync_cmpxchg((ptr), (old), (new), sizeof(*ptr))
 150
 151 #define cmpxchg_local(ptr, old, new)                                    \
 152         __cmpxchg_local((ptr), (old), (new), sizeof(*ptr))
 153 #endif
 154
 155 #ifdef CONFIG_X86_CMPXCHG64
 156 #define cmpxchg64(ptr, o, n)                                            \
 157         ((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \
 158                                          (unsigned long long)(n)))
 159 #define cmpxchg64_local(ptr, o, n)                                      \
 160         ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \
 161                                                (unsigned long long)(n)))
 162 #endif
 163
 164 static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new)
 165 {
 166         u64 prev;
 167         asm volatile(LOCK_PREFIX "cmpxchg8b %1"
 168                      : "=A" (prev),
 169                        "+m" (*ptr)
 170                      : "b" ((u32)new),
 171                        "c" ((u32)(new >> 32)),
 172                        "0" (old)
 173                      : "memory");
 174         return prev;
 175 }
 176
 177 static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new)
 178 {
 179         u64 prev;
 180         asm volatile("cmpxchg8b %1"
 181                      : "=A" (prev),
 182                        "+m" (*ptr)
 183                      : "b" ((u32)new),
 184                        "c" ((u32)(new >> 32)),
 185                        "0" (old)
 186                      : "memory");
 187         return prev;
 188 }
 189
 190 #ifndef CONFIG_X86_CMPXCHG
 191 /*
 192  * Building a kernel capable running on 80386. It may be necessary to
 193  * simulate the cmpxchg on the 80386 CPU. For that purpose we define
 194  * a function for each of the sizes we support.
 195  */
 196
 197 extern unsigned long cmpxchg_386_u8(volatile void *, u8, u8);
 198 extern unsigned long cmpxchg_386_u16(volatile void *, u16, u16);
 199 extern unsigned long cmpxchg_386_u32(volatile void *, u32, u32);
 200
 201 static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old,
 202                                         unsigned long new, int size)
 203 {
 204         switch (size) {
 205         case 1:
 206                 return cmpxchg_386_u8(ptr, old, new);
 207         case 2:
 208                 return cmpxchg_386_u16(ptr, old, new);
 209         case 4:
 210                 return cmpxchg_386_u32(ptr, old, new);
 211         }
 212         return old;
 213 }
 214
 215 #define cmpxchg(ptr, o, n)                                              \
 216 ({                                                                      \
 217         __typeof__(*(ptr)) __ret;                                       \
 218         if (likely(boot_cpu_data.x86 > 3))                              \
 219                 __ret = (__typeof__(*(ptr)))__cmpxchg((ptr),            \
 220                                 (unsigned long)(o), (unsigned long)(n), \
 221                                 sizeof(*(ptr)));                        \
 222         else                                                            \
 223                 __ret = (__typeof__(*(ptr)))cmpxchg_386((ptr),          \
 224                                 (unsigned long)(o), (unsigned long)(n), \
 225                                 sizeof(*(ptr)));                        \
 226         __ret;                                                          \
 227 })
 228 #define cmpxchg_local(ptr, o, n)                                        \
 229 ({                                                                      \
 230         __typeof__(*(ptr)) __ret;                                       \
 231         if (likely(boot_cpu_data.x86 > 3))                              \
 232                 __ret = (__typeof__(*(ptr)))__cmpxchg_local((ptr),      \
 233                                 (unsigned long)(o), (unsigned long)(n), \
 234                                 sizeof(*(ptr)));                        \
 235         else                                                            \
 236                 __ret = (__typeof__(*(ptr)))cmpxchg_386((ptr),          \
 237                                 (unsigned long)(o), (unsigned long)(n), \
 238                                 sizeof(*(ptr)));                        \
 239         __ret;                                                          \
 240 })
 241 #endif
 242
 243 #ifndef CONFIG_X86_CMPXCHG64
 244 /*
 245  * Building a kernel capable running on 80386 and 80486. It may be necessary
 246  * to simulate the cmpxchg8b on the 80386 and 80486 CPU.
 247  */
 248
 249 #define cmpxchg64(ptr, o, n)                                    \
 250 ({                                                              \
 251         __typeof__(*(ptr)) __ret;                               \
 252         __typeof__(*(ptr)) __old = (o);                         \
 253         __typeof__(*(ptr)) __new = (n);                         \
 254         alternative_io(LOCK_PREFIX_HERE                         \
 255                         "call cmpxchg8b_emu",                   \
 256                         "lock; cmpxchg8b (%%esi)" ,             \
 257                        X86_FEATURE_CX8,                         \
 258                        "=A" (__ret),                            \
 259                        "S" ((ptr)), "0" (__old),                \
 260                        "b" ((unsigned int)__new),               \
 261                        "c" ((unsigned int)(__new>>32))          \
 262                        : "memory");                             \
 263         __ret; })
 264
 265
 266 #define cmpxchg64_local(ptr, o, n)                              \
 267 ({                                                              \
 268         __typeof__(*(ptr)) __ret;                               \
 269         __typeof__(*(ptr)) __old = (o);                         \
 270         __typeof__(*(ptr)) __new = (n);                         \
 271         alternative_io("call cmpxchg8b_emu",                    \
 272                        "cmpxchg8b (%%esi)" ,                    \
 273                        X86_FEATURE_CX8,                         \
 274                        "=A" (__ret),                            \
 275                        "S" ((ptr)), "0" (__old),                \
 276                        "b" ((unsigned int)__new),               \
 277                        "c" ((unsigned int)(__new>>32))          \
 278                        : "memory");                             \
 279         __ret; })
 280
 281 #endif
 282
 283 #endif /* _ASM_X86_CMPXCHG_32_H */