arch/s390/include/asm/atomic.h

   1 #ifndef __ARCH_S390_ATOMIC__
   2 #define __ARCH_S390_ATOMIC__
   3
   4 /*
   5  * Copyright 1999,2009 IBM Corp.
   6  * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>,
   7  *            Denis Joseph Barrow,
   8  *            Arnd Bergmann <arndb@de.ibm.com>,
   9  *
  10  * Atomic operations that C can't guarantee us.
  11  * Useful for resource counting etc.
  12  * s390 uses 'Compare And Swap' for atomicity in SMP enviroment.
  13  *
  14  */
  15
  16 #include <linux/compiler.h>
  17 #include <linux/types.h>
  18
  19 #define ATOMIC_INIT(i)  { (i) }
  20
  21 #if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 2)
  22
  23 #define __CS_LOOP(ptr, op_val, op_string) ({                            \
  24         typeof(ptr->counter) old_val, new_val;                          \
  25         asm volatile(                                                   \
  26                 "       l       %0,%2\n"                                \
  27                 "0:     lr      %1,%0\n"                                \
  28                 op_string "     %1,%3\n"                                \
  29                 "       cs      %0,%1,%2\n"                             \
  30                 "       jl      0b"                                     \
  31                 : "=&d" (old_val), "=&d" (new_val),                     \
  32                   "=Q" (((atomic_t *)(ptr))->counter)                   \
  33                 : "d" (op_val),  "Q" (((atomic_t *)(ptr))->counter)     \
  34                 : "cc", "memory");                                      \
  35         new_val;                                                        \
  36 })
  37
  38 #else /* __GNUC__ */
  39
  40 #define __CS_LOOP(ptr, op_val, op_string) ({                            \
  41         typeof(ptr->counter) old_val, new_val;                          \
  42         asm volatile(                                                   \
  43                 "       l       %0,0(%3)\n"                             \
  44                 "0:     lr      %1,%0\n"                                \
  45                 op_string "     %1,%4\n"                                \
  46                 "       cs      %0,%1,0(%3)\n"                          \
  47                 "       jl      0b"                                     \
  48                 : "=&d" (old_val), "=&d" (new_val),                     \
  49                   "=m" (((atomic_t *)(ptr))->counter)                   \
  50                 : "a" (ptr), "d" (op_val),                              \
  51                   "m" (((atomic_t *)(ptr))->counter)                    \
  52                 : "cc", "memory");                                      \
  53         new_val;                                                        \
  54 })
  55
  56 #endif /* __GNUC__ */
  57
  58 static inline int atomic_read(const atomic_t *v)
  59 {
  60         barrier();
  61         return v->counter;
  62 }
  63
  64 static inline void atomic_set(atomic_t *v, int i)
  65 {
  66         v->counter = i;
  67         barrier();
  68 }
  69
  70 static inline int atomic_add_return(int i, atomic_t *v)
  71 {
  72         return __CS_LOOP(v, i, "ar");
  73 }
  74 #define atomic_add(_i, _v)              atomic_add_return(_i, _v)
  75 #define atomic_add_negative(_i, _v)     (atomic_add_return(_i, _v) < 0)
  76 #define atomic_inc(_v)                  atomic_add_return(1, _v)
  77 #define atomic_inc_return(_v)           atomic_add_return(1, _v)
  78 #define atomic_inc_and_test(_v)         (atomic_add_return(1, _v) == 0)
  79
  80 static inline int atomic_sub_return(int i, atomic_t *v)
  81 {
  82         return __CS_LOOP(v, i, "sr");
  83 }
  84 #define atomic_sub(_i, _v)              atomic_sub_return(_i, _v)
  85 #define atomic_sub_and_test(_i, _v)     (atomic_sub_return(_i, _v) == 0)
  86 #define atomic_dec(_v)                  atomic_sub_return(1, _v)
  87 #define atomic_dec_return(_v)           atomic_sub_return(1, _v)
  88 #define atomic_dec_and_test(_v)         (atomic_sub_return(1, _v) == 0)
  89
  90 static inline void atomic_clear_mask(unsigned long mask, atomic_t *v)
  91 {
  92         __CS_LOOP(v, ~mask, "nr");
  93 }
  94
  95 static inline void atomic_set_mask(unsigned long mask, atomic_t *v)
  96 {
  97         __CS_LOOP(v, mask, "or");
  98 }
  99
 100 #define atomic_xchg(v, new) (xchg(&((v)->counter), new))
 101
 102 static inline int atomic_cmpxchg(atomic_t *v, int old, int new)
 103 {
 104 #if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 2)
 105         asm volatile(
 106                 "       cs      %0,%2,%1"
 107                 : "+d" (old), "=Q" (v->counter)
 108                 : "d" (new), "Q" (v->counter)
 109                 : "cc", "memory");
 110 #else /* __GNUC__ */
 111         asm volatile(
 112                 "       cs      %0,%3,0(%2)"
 113                 : "+d" (old), "=m" (v->counter)
 114                 : "a" (v), "d" (new), "m" (v->counter)
 115                 : "cc", "memory");
 116 #endif /* __GNUC__ */
 117         return old;
 118 }
 119
 120 static inline int atomic_add_unless(atomic_t *v, int a, int u)
 121 {
 122         int c, old;
 123         c = atomic_read(v);
 124         for (;;) {
 125                 if (unlikely(c == u))
 126                         break;
 127                 old = atomic_cmpxchg(v, c, c + a);
 128                 if (likely(old == c))
 129                         break;
 130                 c = old;
 131         }
 132         return c != u;
 133 }
 134
 135 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
 136
 137 #undef __CS_LOOP
 138
 139 #define ATOMIC64_INIT(i)  { (i) }
 140
 141 #ifdef CONFIG_64BIT
 142
 143 #if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 2)
 144
 145 #define __CSG_LOOP(ptr, op_val, op_string) ({                           \
 146         typeof(ptr->counter) old_val, new_val;                          \
 147         asm volatile(                                                   \
 148                 "       lg      %0,%2\n"                                \
 149                 "0:     lgr     %1,%0\n"                                \
 150                 op_string "     %1,%3\n"                                \
 151                 "       csg     %0,%1,%2\n"                             \
 152                 "       jl      0b"                                     \
 153                 : "=&d" (old_val), "=&d" (new_val),                     \
 154                   "=Q" (((atomic_t *)(ptr))->counter)                   \
 155                 : "d" (op_val), "Q" (((atomic_t *)(ptr))->counter)      \
 156                 : "cc", "memory");                                      \
 157         new_val;                                                        \
 158 })
 159
 160 #else /* __GNUC__ */
 161
 162 #define __CSG_LOOP(ptr, op_val, op_string) ({                           \
 163         typeof(ptr->counter) old_val, new_val;                          \
 164         asm volatile(                                                   \
 165                 "       lg      %0,0(%3)\n"                             \
 166                 "0:     lgr     %1,%0\n"                                \
 167                 op_string "     %1,%4\n"                                \
 168                 "       csg     %0,%1,0(%3)\n"                          \
 169                 "       jl      0b"                                     \
 170                 : "=&d" (old_val), "=&d" (new_val),                     \
 171                   "=m" (((atomic_t *)(ptr))->counter)                   \
 172                 : "a" (ptr), "d" (op_val),                              \
 173                   "m" (((atomic_t *)(ptr))->counter)                    \
 174                 : "cc", "memory");                                      \
 175         new_val;                                                        \
 176 })
 177
 178 #endif /* __GNUC__ */
 179
 180 static inline long long atomic64_read(const atomic64_t *v)
 181 {
 182         barrier();
 183         return v->counter;
 184 }
 185
 186 static inline void atomic64_set(atomic64_t *v, long long i)
 187 {
 188         v->counter = i;
 189         barrier();
 190 }
 191
 192 static inline long long atomic64_add_return(long long i, atomic64_t *v)
 193 {
 194         return __CSG_LOOP(v, i, "agr");
 195 }
 196
 197 static inline long long atomic64_sub_return(long long i, atomic64_t *v)
 198 {
 199         return __CSG_LOOP(v, i, "sgr");
 200 }
 201
 202 static inline void atomic64_clear_mask(unsigned long mask, atomic64_t *v)
 203 {
 204         __CSG_LOOP(v, ~mask, "ngr");
 205 }
 206
 207 static inline void atomic64_set_mask(unsigned long mask, atomic64_t *v)
 208 {
 209         __CSG_LOOP(v, mask, "ogr");
 210 }
 211
 212 #define atomic64_xchg(v, new) (xchg(&((v)->counter), new))
 213
 214 static inline long long atomic64_cmpxchg(atomic64_t *v,
 215                                              long long old, long long new)
 216 {
 217 #if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 2)
 218         asm volatile(
 219                 "       csg     %0,%2,%1"
 220                 : "+d" (old), "=Q" (v->counter)
 221                 : "d" (new), "Q" (v->counter)
 222                 : "cc", "memory");
 223 #else /* __GNUC__ */
 224         asm volatile(
 225                 "       csg     %0,%3,0(%2)"
 226                 : "+d" (old), "=m" (v->counter)
 227                 : "a" (v), "d" (new), "m" (v->counter)
 228                 : "cc", "memory");
 229 #endif /* __GNUC__ */
 230         return old;
 231 }
 232
 233 #undef __CSG_LOOP
 234
 235 #else /* CONFIG_64BIT */
 236
 237 typedef struct {
 238         long long counter;
 239 } atomic64_t;
 240
 241 static inline long long atomic64_read(const atomic64_t *v)
 242 {
 243         register_pair rp;
 244
 245         asm volatile(
 246                 "       lm      %0,%N0,0(%1)"
 247                 : "=&d" (rp)
 248                 : "a" (&v->counter), "m" (v->counter)
 249                 );
 250         return rp.pair;
 251 }
 252
 253 static inline void atomic64_set(atomic64_t *v, long long i)
 254 {
 255         register_pair rp = {.pair = i};
 256
 257         asm volatile(
 258                 "       stm     %1,%N1,0(%2)"
 259                 : "=m" (v->counter)
 260                 : "d" (rp), "a" (&v->counter)
 261                 );
 262 }
 263
 264 static inline long long atomic64_xchg(atomic64_t *v, long long new)
 265 {
 266         register_pair rp_new = {.pair = new};
 267         register_pair rp_old;
 268
 269         asm volatile(
 270                 "       lm      %0,%N0,0(%2)\n"
 271                 "0:     cds     %0,%3,0(%2)\n"
 272                 "       jl      0b\n"
 273                 : "=&d" (rp_old), "+m" (v->counter)
 274                 : "a" (&v->counter), "d" (rp_new)
 275                 : "cc");
 276         return rp_old.pair;
 277 }
 278
 279 static inline long long atomic64_cmpxchg(atomic64_t *v,
 280                                          long long old, long long new)
 281 {
 282         register_pair rp_old = {.pair = old};
 283         register_pair rp_new = {.pair = new};
 284
 285         asm volatile(
 286                 "       cds     %0,%3,0(%2)"
 287                 : "+&d" (rp_old), "+m" (v->counter)
 288                 : "a" (&v->counter), "d" (rp_new)
 289                 : "cc");
 290         return rp_old.pair;
 291 }
 292
 293
 294 static inline long long atomic64_add_return(long long i, atomic64_t *v)
 295 {
 296         long long old, new;
 297
 298         do {
 299                 old = atomic64_read(v);
 300                 new = old + i;
 301         } while (atomic64_cmpxchg(v, old, new) != old);
 302         return new;
 303 }
 304
 305 static inline long long atomic64_sub_return(long long i, atomic64_t *v)
 306 {
 307         long long old, new;
 308
 309         do {
 310                 old = atomic64_read(v);
 311                 new = old - i;
 312         } while (atomic64_cmpxchg(v, old, new) != old);
 313         return new;
 314 }
 315
 316 static inline void atomic64_set_mask(unsigned long long mask, atomic64_t *v)
 317 {
 318         long long old, new;
 319
 320         do {
 321                 old = atomic64_read(v);
 322                 new = old | mask;
 323         } while (atomic64_cmpxchg(v, old, new) != old);
 324 }
 325
 326 static inline void atomic64_clear_mask(unsigned long long mask, atomic64_t *v)
 327 {
 328         long long old, new;
 329
 330         do {
 331                 old = atomic64_read(v);
 332                 new = old & mask;
 333         } while (atomic64_cmpxchg(v, old, new) != old);
 334 }
 335
 336 #endif /* CONFIG_64BIT */
 337
 338 static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u)
 339 {
 340         long long c, old;
 341         c = atomic64_read(v);
 342         for (;;) {
 343                 if (unlikely(c == u))
 344                         break;
 345                 old = atomic64_cmpxchg(v, c, c + a);
 346                 if (likely(old == c))
 347                         break;
 348                 c = old;
 349         }
 350         return c != u;
 351 }
 352
 353 #define atomic64_add(_i, _v)            atomic64_add_return(_i, _v)
 354 #define atomic64_add_negative(_i, _v)   (atomic64_add_return(_i, _v) < 0)
 355 #define atomic64_inc(_v)                atomic64_add_return(1, _v)
 356 #define atomic64_inc_return(_v)         atomic64_add_return(1, _v)
 357 #define atomic64_inc_and_test(_v)       (atomic64_add_return(1, _v) == 0)
 358 #define atomic64_sub(_i, _v)            atomic64_sub_return(_i, _v)
 359 #define atomic64_sub_and_test(_i, _v)   (atomic64_sub_return(_i, _v) == 0)
 360 #define atomic64_dec(_v)                atomic64_sub_return(1, _v)
 361 #define atomic64_dec_return(_v)         atomic64_sub_return(1, _v)
 362 #define atomic64_dec_and_test(_v)       (atomic64_sub_return(1, _v) == 0)
 363 #define atomic64_inc_not_zero(v)        atomic64_add_unless((v), 1, 0)
 364
 365 #define smp_mb__before_atomic_dec()     smp_mb()
 366 #define smp_mb__after_atomic_dec()      smp_mb()
 367 #define smp_mb__before_atomic_inc()     smp_mb()
 368 #define smp_mb__after_atomic_inc()      smp_mb()
 369
 370 #include <asm-generic/atomic-long.h>
 371
 372 #endif /* __ARCH_S390_ATOMIC__  */