x86: Add support for cmpxchg_double
[pandora-kernel.git] / arch / x86 / include / asm / cmpxchg_32.h
1 #ifndef _ASM_X86_CMPXCHG_32_H
2 #define _ASM_X86_CMPXCHG_32_H
3
4 #include <linux/bitops.h> /* for LOCK_PREFIX */
5
6 /*
7  * Note: if you use set64_bit(), __cmpxchg64(), or their variants, you
8  *       you need to test for the feature in boot_cpu_data.
9  */
10
11 extern void __xchg_wrong_size(void);
12
13 /*
14  * Note: no "lock" prefix even on SMP: xchg always implies lock anyway.
15  * Since this is generally used to protect other memory information, we
16  * use "asm volatile" and "memory" clobbers to prevent gcc from moving
17  * information around.
18  */
19 #define __xchg(x, ptr, size)                                            \
20 ({                                                                      \
21         __typeof(*(ptr)) __x = (x);                                     \
22         switch (size) {                                                 \
23         case 1:                                                         \
24         {                                                               \
25                 volatile u8 *__ptr = (volatile u8 *)(ptr);              \
26                 asm volatile("xchgb %0,%1"                              \
27                              : "=q" (__x), "+m" (*__ptr)                \
28                              : "0" (__x)                                \
29                              : "memory");                               \
30                 break;                                                  \
31         }                                                               \
32         case 2:                                                         \
33         {                                                               \
34                 volatile u16 *__ptr = (volatile u16 *)(ptr);            \
35                 asm volatile("xchgw %0,%1"                              \
36                              : "=r" (__x), "+m" (*__ptr)                \
37                              : "0" (__x)                                \
38                              : "memory");                               \
39                 break;                                                  \
40         }                                                               \
41         case 4:                                                         \
42         {                                                               \
43                 volatile u32 *__ptr = (volatile u32 *)(ptr);            \
44                 asm volatile("xchgl %0,%1"                              \
45                              : "=r" (__x), "+m" (*__ptr)                \
46                              : "0" (__x)                                \
47                              : "memory");                               \
48                 break;                                                  \
49         }                                                               \
50         default:                                                        \
51                 __xchg_wrong_size();                                    \
52         }                                                               \
53         __x;                                                            \
54 })
55
56 #define xchg(ptr, v)                                                    \
57         __xchg((v), (ptr), sizeof(*ptr))
58
59 /*
60  * CMPXCHG8B only writes to the target if we had the previous
61  * value in registers, otherwise it acts as a read and gives us the
62  * "new previous" value.  That is why there is a loop.  Preloading
63  * EDX:EAX is a performance optimization: in the common case it means
64  * we need only one locked operation.
65  *
66  * A SIMD/3DNOW!/MMX/FPU 64-bit store here would require at the very
67  * least an FPU save and/or %cr0.ts manipulation.
68  *
69  * cmpxchg8b must be used with the lock prefix here to allow the
70  * instruction to be executed atomically.  We need to have the reader
71  * side to see the coherent 64bit value.
72  */
73 static inline void set_64bit(volatile u64 *ptr, u64 value)
74 {
75         u32 low  = value;
76         u32 high = value >> 32;
77         u64 prev = *ptr;
78
79         asm volatile("\n1:\t"
80                      LOCK_PREFIX "cmpxchg8b %0\n\t"
81                      "jnz 1b"
82                      : "=m" (*ptr), "+A" (prev)
83                      : "b" (low), "c" (high)
84                      : "memory");
85 }
86
87 extern void __cmpxchg_wrong_size(void);
88
89 /*
90  * Atomic compare and exchange.  Compare OLD with MEM, if identical,
91  * store NEW in MEM.  Return the initial value in MEM.  Success is
92  * indicated by comparing RETURN with OLD.
93  */
94 #define __raw_cmpxchg(ptr, old, new, size, lock)                        \
95 ({                                                                      \
96         __typeof__(*(ptr)) __ret;                                       \
97         __typeof__(*(ptr)) __old = (old);                               \
98         __typeof__(*(ptr)) __new = (new);                               \
99         switch (size) {                                                 \
100         case 1:                                                         \
101         {                                                               \
102                 volatile u8 *__ptr = (volatile u8 *)(ptr);              \
103                 asm volatile(lock "cmpxchgb %2,%1"                      \
104                              : "=a" (__ret), "+m" (*__ptr)              \
105                              : "q" (__new), "0" (__old)                 \
106                              : "memory");                               \
107                 break;                                                  \
108         }                                                               \
109         case 2:                                                         \
110         {                                                               \
111                 volatile u16 *__ptr = (volatile u16 *)(ptr);            \
112                 asm volatile(lock "cmpxchgw %2,%1"                      \
113                              : "=a" (__ret), "+m" (*__ptr)              \
114                              : "r" (__new), "0" (__old)                 \
115                              : "memory");                               \
116                 break;                                                  \
117         }                                                               \
118         case 4:                                                         \
119         {                                                               \
120                 volatile u32 *__ptr = (volatile u32 *)(ptr);            \
121                 asm volatile(lock "cmpxchgl %2,%1"                      \
122                              : "=a" (__ret), "+m" (*__ptr)              \
123                              : "r" (__new), "0" (__old)                 \
124                              : "memory");                               \
125                 break;                                                  \
126         }                                                               \
127         default:                                                        \
128                 __cmpxchg_wrong_size();                                 \
129         }                                                               \
130         __ret;                                                          \
131 })
132
133 #define __cmpxchg(ptr, old, new, size)                                  \
134         __raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX)
135
136 #define __sync_cmpxchg(ptr, old, new, size)                             \
137         __raw_cmpxchg((ptr), (old), (new), (size), "lock; ")
138
139 #define __cmpxchg_local(ptr, old, new, size)                            \
140         __raw_cmpxchg((ptr), (old), (new), (size), "")
141
142 #ifdef CONFIG_X86_CMPXCHG
143 #define __HAVE_ARCH_CMPXCHG 1
144
145 #define cmpxchg(ptr, old, new)                                          \
146         __cmpxchg((ptr), (old), (new), sizeof(*ptr))
147
148 #define sync_cmpxchg(ptr, old, new)                                     \
149         __sync_cmpxchg((ptr), (old), (new), sizeof(*ptr))
150
151 #define cmpxchg_local(ptr, old, new)                                    \
152         __cmpxchg_local((ptr), (old), (new), sizeof(*ptr))
153 #endif
154
155 #ifdef CONFIG_X86_CMPXCHG64
156 #define cmpxchg64(ptr, o, n)                                            \
157         ((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \
158                                          (unsigned long long)(n)))
159 #define cmpxchg64_local(ptr, o, n)                                      \
160         ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \
161                                                (unsigned long long)(n)))
162 #endif
163
164 static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new)
165 {
166         u64 prev;
167         asm volatile(LOCK_PREFIX "cmpxchg8b %1"
168                      : "=A" (prev),
169                        "+m" (*ptr)
170                      : "b" ((u32)new),
171                        "c" ((u32)(new >> 32)),
172                        "0" (old)
173                      : "memory");
174         return prev;
175 }
176
177 static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new)
178 {
179         u64 prev;
180         asm volatile("cmpxchg8b %1"
181                      : "=A" (prev),
182                        "+m" (*ptr)
183                      : "b" ((u32)new),
184                        "c" ((u32)(new >> 32)),
185                        "0" (old)
186                      : "memory");
187         return prev;
188 }
189
190 #ifndef CONFIG_X86_CMPXCHG
191 /*
192  * Building a kernel capable running on 80386. It may be necessary to
193  * simulate the cmpxchg on the 80386 CPU. For that purpose we define
194  * a function for each of the sizes we support.
195  */
196
197 extern unsigned long cmpxchg_386_u8(volatile void *, u8, u8);
198 extern unsigned long cmpxchg_386_u16(volatile void *, u16, u16);
199 extern unsigned long cmpxchg_386_u32(volatile void *, u32, u32);
200
201 static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old,
202                                         unsigned long new, int size)
203 {
204         switch (size) {
205         case 1:
206                 return cmpxchg_386_u8(ptr, old, new);
207         case 2:
208                 return cmpxchg_386_u16(ptr, old, new);
209         case 4:
210                 return cmpxchg_386_u32(ptr, old, new);
211         }
212         return old;
213 }
214
215 #define cmpxchg(ptr, o, n)                                              \
216 ({                                                                      \
217         __typeof__(*(ptr)) __ret;                                       \
218         if (likely(boot_cpu_data.x86 > 3))                              \
219                 __ret = (__typeof__(*(ptr)))__cmpxchg((ptr),            \
220                                 (unsigned long)(o), (unsigned long)(n), \
221                                 sizeof(*(ptr)));                        \
222         else                                                            \
223                 __ret = (__typeof__(*(ptr)))cmpxchg_386((ptr),          \
224                                 (unsigned long)(o), (unsigned long)(n), \
225                                 sizeof(*(ptr)));                        \
226         __ret;                                                          \
227 })
228 #define cmpxchg_local(ptr, o, n)                                        \
229 ({                                                                      \
230         __typeof__(*(ptr)) __ret;                                       \
231         if (likely(boot_cpu_data.x86 > 3))                              \
232                 __ret = (__typeof__(*(ptr)))__cmpxchg_local((ptr),      \
233                                 (unsigned long)(o), (unsigned long)(n), \
234                                 sizeof(*(ptr)));                        \
235         else                                                            \
236                 __ret = (__typeof__(*(ptr)))cmpxchg_386((ptr),          \
237                                 (unsigned long)(o), (unsigned long)(n), \
238                                 sizeof(*(ptr)));                        \
239         __ret;                                                          \
240 })
241 #endif
242
243 #ifndef CONFIG_X86_CMPXCHG64
244 /*
245  * Building a kernel capable running on 80386 and 80486. It may be necessary
246  * to simulate the cmpxchg8b on the 80386 and 80486 CPU.
247  */
248
249 #define cmpxchg64(ptr, o, n)                                    \
250 ({                                                              \
251         __typeof__(*(ptr)) __ret;                               \
252         __typeof__(*(ptr)) __old = (o);                         \
253         __typeof__(*(ptr)) __new = (n);                         \
254         alternative_io(LOCK_PREFIX_HERE                         \
255                         "call cmpxchg8b_emu",                   \
256                         "lock; cmpxchg8b (%%esi)" ,             \
257                        X86_FEATURE_CX8,                         \
258                        "=A" (__ret),                            \
259                        "S" ((ptr)), "0" (__old),                \
260                        "b" ((unsigned int)__new),               \
261                        "c" ((unsigned int)(__new>>32))          \
262                        : "memory");                             \
263         __ret; })
264
265
266 #define cmpxchg64_local(ptr, o, n)                              \
267 ({                                                              \
268         __typeof__(*(ptr)) __ret;                               \
269         __typeof__(*(ptr)) __old = (o);                         \
270         __typeof__(*(ptr)) __new = (n);                         \
271         alternative_io("call cmpxchg8b_emu",                    \
272                        "cmpxchg8b (%%esi)" ,                    \
273                        X86_FEATURE_CX8,                         \
274                        "=A" (__ret),                            \
275                        "S" ((ptr)), "0" (__old),                \
276                        "b" ((unsigned int)__new),               \
277                        "c" ((unsigned int)(__new>>32))          \
278                        : "memory");                             \
279         __ret; })
280
281 #endif
282
283 #define cmpxchg8b(ptr, o1, o2, n1, n2)                          \
284 ({                                                              \
285         char __ret;                                             \
286         __typeof__(o2) __dummy;                                 \
287         __typeof__(*(ptr)) __old1 = (o1);                       \
288         __typeof__(o2) __old2 = (o2);                           \
289         __typeof__(*(ptr)) __new1 = (n1);                       \
290         __typeof__(o2) __new2 = (n2);                           \
291         asm volatile(LOCK_PREFIX "cmpxchg8b %2; setz %1"        \
292                        : "=d"(__dummy), "=a" (__ret), "+m" (*ptr)\
293                        : "a" (__old1), "d"(__old2),             \
294                          "b" (__new1), "c" (__new2)             \
295                        : "memory");                             \
296         __ret; })
297
298
299 #define cmpxchg8b_local(ptr, o1, o2, n1, n2)                    \
300 ({                                                              \
301         char __ret;                                             \
302         __typeof__(o2) __dummy;                                 \
303         __typeof__(*(ptr)) __old1 = (o1);                       \
304         __typeof__(o2) __old2 = (o2);                           \
305         __typeof__(*(ptr)) __new1 = (n1);                       \
306         __typeof__(o2) __new2 = (n2);                           \
307         asm volatile("cmpxchg8b %2; setz %1"                    \
308                        : "=d"(__dummy), "=a"(__ret), "+m" (*ptr)\
309                        : "a" (__old), "d"(__old2),              \
310                          "b" (__new1), "c" (__new2),            \
311                        : "memory");                             \
312         __ret; })
313
314
315 #define cmpxchg_double(ptr, o1, o2, n1, n2)                             \
316 ({                                                                      \
317         BUILD_BUG_ON(sizeof(*(ptr)) != 4);                              \
318         VM_BUG_ON((unsigned long)(ptr) % 8);                            \
319         cmpxchg8b((ptr), (o1), (o2), (n1), (n2));                       \
320 })
321
322 #define cmpxchg_double_local(ptr, o1, o2, n1, n2)                       \
323 ({                                                                      \
324        BUILD_BUG_ON(sizeof(*(ptr)) != 4);                               \
325        VM_BUG_ON((unsigned long)(ptr) % 8);                             \
326        cmpxchg16b_local((ptr), (o1), (o2), (n1), (n2));                 \
327 })
328
329 #define system_has_cmpxchg_double() cpu_has_cx8
330
331 #endif /* _ASM_X86_CMPXCHG_32_H */