kernel/spinlock.c

   1 /*
   2  * Copyright (2004) Linus Torvalds
   3  *
   4  * Author: Zwane Mwaikambo <zwane@fsmlabs.com>
   5  *
   6  * Copyright (2004, 2005) Ingo Molnar
   7  *
   8  * This file contains the spinlock/rwlock implementations for the
   9  * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
  10  *
  11  * Note that some architectures have special knowledge about the
  12  * stack frames of these functions in their profile_pc. If you
  13  * change anything significant here that could change the stack
  14  * frame contact the architecture maintainers.
  15  */
  16
  17 #include <linux/linkage.h>
  18 #include <linux/preempt.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/debug_locks.h>
  22 #include <linux/module.h>
  23
  24 /*
  25  * Generic declaration of the raw read_trylock() function,
  26  * architectures are supposed to optimize this:
  27  */
  28 int __lockfunc generic__raw_read_trylock(raw_rwlock_t *lock)
  29 {
  30         __raw_read_lock(lock);
  31         return 1;
  32 }
  33 EXPORT_SYMBOL(generic__raw_read_trylock);
  34
  35 int __lockfunc _spin_trylock(spinlock_t *lock)
  36 {
  37         preempt_disable();
  38         if (_raw_spin_trylock(lock)) {
  39                 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
  40                 return 1;
  41         }
  42
  43         preempt_enable();
  44         return 0;
  45 }
  46 EXPORT_SYMBOL(_spin_trylock);
  47
  48 int __lockfunc _read_trylock(rwlock_t *lock)
  49 {
  50         preempt_disable();
  51         if (_raw_read_trylock(lock)) {
  52                 rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_);
  53                 return 1;
  54         }
  55
  56         preempt_enable();
  57         return 0;
  58 }
  59 EXPORT_SYMBOL(_read_trylock);
  60
  61 int __lockfunc _write_trylock(rwlock_t *lock)
  62 {
  63         preempt_disable();
  64         if (_raw_write_trylock(lock)) {
  65                 rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_);
  66                 return 1;
  67         }
  68
  69         preempt_enable();
  70         return 0;
  71 }
  72 EXPORT_SYMBOL(_write_trylock);
  73
  74 /*
  75  * If lockdep is enabled then we use the non-preemption spin-ops
  76  * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
  77  * not re-enabled during lock-acquire (which the preempt-spin-ops do):
  78  */
  79 #if !defined(CONFIG_PREEMPT) || !defined(CONFIG_SMP) || \
  80         defined(CONFIG_DEBUG_LOCK_ALLOC)
  81
  82 void __lockfunc _read_lock(rwlock_t *lock)
  83 {
  84         preempt_disable();
  85         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
  86         _raw_read_lock(lock);
  87 }
  88 EXPORT_SYMBOL(_read_lock);
  89
  90 unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
  91 {
  92         unsigned long flags;
  93
  94         local_irq_save(flags);
  95         preempt_disable();
  96         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
  97         /*
  98          * On lockdep we dont want the hand-coded irq-enable of
  99          * _raw_spin_lock_flags() code, because lockdep assumes
 100          * that interrupts are not re-enabled during lock-acquire:
 101          */
 102 #ifdef CONFIG_PROVE_LOCKING
 103         _raw_spin_lock(lock);
 104 #else
 105         _raw_spin_lock_flags(lock, &flags);
 106 #endif
 107         return flags;
 108 }
 109 EXPORT_SYMBOL(_spin_lock_irqsave);
 110
 111 void __lockfunc _spin_lock_irq(spinlock_t *lock)
 112 {
 113         local_irq_disable();
 114         preempt_disable();
 115         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 116         _raw_spin_lock(lock);
 117 }
 118 EXPORT_SYMBOL(_spin_lock_irq);
 119
 120 void __lockfunc _spin_lock_bh(spinlock_t *lock)
 121 {
 122         local_bh_disable();
 123         preempt_disable();
 124         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 125         _raw_spin_lock(lock);
 126 }
 127 EXPORT_SYMBOL(_spin_lock_bh);
 128
 129 unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
 130 {
 131         unsigned long flags;
 132
 133         local_irq_save(flags);
 134         preempt_disable();
 135         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
 136         _raw_read_lock(lock);
 137         return flags;
 138 }
 139 EXPORT_SYMBOL(_read_lock_irqsave);
 140
 141 void __lockfunc _read_lock_irq(rwlock_t *lock)
 142 {
 143         local_irq_disable();
 144         preempt_disable();
 145         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
 146         _raw_read_lock(lock);
 147 }
 148 EXPORT_SYMBOL(_read_lock_irq);
 149
 150 void __lockfunc _read_lock_bh(rwlock_t *lock)
 151 {
 152         local_bh_disable();
 153         preempt_disable();
 154         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
 155         _raw_read_lock(lock);
 156 }
 157 EXPORT_SYMBOL(_read_lock_bh);
 158
 159 unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
 160 {
 161         unsigned long flags;
 162
 163         local_irq_save(flags);
 164         preempt_disable();
 165         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 166         _raw_write_lock(lock);
 167         return flags;
 168 }
 169 EXPORT_SYMBOL(_write_lock_irqsave);
 170
 171 void __lockfunc _write_lock_irq(rwlock_t *lock)
 172 {
 173         local_irq_disable();
 174         preempt_disable();
 175         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 176         _raw_write_lock(lock);
 177 }
 178 EXPORT_SYMBOL(_write_lock_irq);
 179
 180 void __lockfunc _write_lock_bh(rwlock_t *lock)
 181 {
 182         local_bh_disable();
 183         preempt_disable();
 184         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 185         _raw_write_lock(lock);
 186 }
 187 EXPORT_SYMBOL(_write_lock_bh);
 188
 189 void __lockfunc _spin_lock(spinlock_t *lock)
 190 {
 191         preempt_disable();
 192         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 193         _raw_spin_lock(lock);
 194 }
 195
 196 EXPORT_SYMBOL(_spin_lock);
 197
 198 void __lockfunc _write_lock(rwlock_t *lock)
 199 {
 200         preempt_disable();
 201         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
 202         _raw_write_lock(lock);
 203 }
 204
 205 EXPORT_SYMBOL(_write_lock);
 206
 207 #else /* CONFIG_PREEMPT: */
 208
 209 /*
 210  * This could be a long-held lock. We both prepare to spin for a long
 211  * time (making _this_ CPU preemptable if possible), and we also signal
 212  * towards that other CPU that it should break the lock ASAP.
 213  *
 214  * (We do this in a function because inlining it would be excessive.)
 215  */
 216
 217 #define BUILD_LOCK_OPS(op, locktype)                                    \
 218 void __lockfunc _##op##_lock(locktype##_t *lock)                        \
 219 {                                                                       \
 220         for (;;) {                                                      \
 221                 preempt_disable();                                      \
 222                 if (likely(_raw_##op##_trylock(lock)))                  \
 223                         break;                                          \
 224                 preempt_enable();                                       \
 225                                                                         \
 226                 if (!(lock)->break_lock)                                \
 227                         (lock)->break_lock = 1;                         \
 228                 while (!op##_can_lock(lock) && (lock)->break_lock)      \
 229                         cpu_relax();                                    \
 230         }                                                               \
 231         (lock)->break_lock = 0;                                         \
 232 }                                                                       \
 233                                                                         \
 234 EXPORT_SYMBOL(_##op##_lock);                                            \
 235                                                                         \
 236 unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock)       \
 237 {                                                                       \
 238         unsigned long flags;                                            \
 239                                                                         \
 240         for (;;) {                                                      \
 241                 preempt_disable();                                      \
 242                 local_irq_save(flags);                                  \
 243                 if (likely(_raw_##op##_trylock(lock)))                  \
 244                         break;                                          \
 245                 local_irq_restore(flags);                               \
 246                 preempt_enable();                                       \
 247                                                                         \
 248                 if (!(lock)->break_lock)                                \
 249                         (lock)->break_lock = 1;                         \
 250                 while (!op##_can_lock(lock) && (lock)->break_lock)      \
 251                         cpu_relax();                                    \
 252         }                                                               \
 253         (lock)->break_lock = 0;                                         \
 254         return flags;                                                   \
 255 }                                                                       \
 256                                                                         \
 257 EXPORT_SYMBOL(_##op##_lock_irqsave);                                    \
 258                                                                         \
 259 void __lockfunc _##op##_lock_irq(locktype##_t *lock)                    \
 260 {                                                                       \
 261         _##op##_lock_irqsave(lock);                                     \
 262 }                                                                       \
 263                                                                         \
 264 EXPORT_SYMBOL(_##op##_lock_irq);                                        \
 265                                                                         \
 266 void __lockfunc _##op##_lock_bh(locktype##_t *lock)                     \
 267 {                                                                       \
 268         unsigned long flags;                                            \
 269                                                                         \
 270         /*                                                      */      \
 271         /* Careful: we must exclude softirqs too, hence the     */      \
 272         /* irq-disabling. We use the generic preemption-aware   */      \
 273         /* function:                                            */      \
 274         /**/                                                            \
 275         flags = _##op##_lock_irqsave(lock);                             \
 276         local_bh_disable();                                             \
 277         local_irq_restore(flags);                                       \
 278 }                                                                       \
 279                                                                         \
 280 EXPORT_SYMBOL(_##op##_lock_bh)
 281
 282 /*
 283  * Build preemption-friendly versions of the following
 284  * lock-spinning functions:
 285  *
 286  *         _[spin|read|write]_lock()
 287  *         _[spin|read|write]_lock_irq()
 288  *         _[spin|read|write]_lock_irqsave()
 289  *         _[spin|read|write]_lock_bh()
 290  */
 291 BUILD_LOCK_OPS(spin, spinlock);
 292 BUILD_LOCK_OPS(read, rwlock);
 293 BUILD_LOCK_OPS(write, rwlock);
 294
 295 #endif /* CONFIG_PREEMPT */
 296
 297 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 298
 299 void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass)
 300 {
 301         preempt_disable();
 302         spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
 303         _raw_spin_lock(lock);
 304 }
 305
 306 EXPORT_SYMBOL(_spin_lock_nested);
 307
 308 #endif
 309
 310 void __lockfunc _spin_unlock(spinlock_t *lock)
 311 {
 312         spin_release(&lock->dep_map, 1, _RET_IP_);
 313         _raw_spin_unlock(lock);
 314         preempt_enable();
 315 }
 316 EXPORT_SYMBOL(_spin_unlock);
 317
 318 void __lockfunc _write_unlock(rwlock_t *lock)
 319 {
 320         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 321         _raw_write_unlock(lock);
 322         preempt_enable();
 323 }
 324 EXPORT_SYMBOL(_write_unlock);
 325
 326 void __lockfunc _read_unlock(rwlock_t *lock)
 327 {
 328         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 329         _raw_read_unlock(lock);
 330         preempt_enable();
 331 }
 332 EXPORT_SYMBOL(_read_unlock);
 333
 334 void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
 335 {
 336         spin_release(&lock->dep_map, 1, _RET_IP_);
 337         _raw_spin_unlock(lock);
 338         local_irq_restore(flags);
 339         preempt_enable();
 340 }
 341 EXPORT_SYMBOL(_spin_unlock_irqrestore);
 342
 343 void __lockfunc _spin_unlock_irq(spinlock_t *lock)
 344 {
 345         spin_release(&lock->dep_map, 1, _RET_IP_);
 346         _raw_spin_unlock(lock);
 347         local_irq_enable();
 348         preempt_enable();
 349 }
 350 EXPORT_SYMBOL(_spin_unlock_irq);
 351
 352 void __lockfunc _spin_unlock_bh(spinlock_t *lock)
 353 {
 354         spin_release(&lock->dep_map, 1, _RET_IP_);
 355         _raw_spin_unlock(lock);
 356         preempt_enable_no_resched();
 357         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 358 }
 359 EXPORT_SYMBOL(_spin_unlock_bh);
 360
 361 void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 362 {
 363         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 364         _raw_read_unlock(lock);
 365         local_irq_restore(flags);
 366         preempt_enable();
 367 }
 368 EXPORT_SYMBOL(_read_unlock_irqrestore);
 369
 370 void __lockfunc _read_unlock_irq(rwlock_t *lock)
 371 {
 372         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 373         _raw_read_unlock(lock);
 374         local_irq_enable();
 375         preempt_enable();
 376 }
 377 EXPORT_SYMBOL(_read_unlock_irq);
 378
 379 void __lockfunc _read_unlock_bh(rwlock_t *lock)
 380 {
 381         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 382         _raw_read_unlock(lock);
 383         preempt_enable_no_resched();
 384         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 385 }
 386 EXPORT_SYMBOL(_read_unlock_bh);
 387
 388 void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 389 {
 390         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 391         _raw_write_unlock(lock);
 392         local_irq_restore(flags);
 393         preempt_enable();
 394 }
 395 EXPORT_SYMBOL(_write_unlock_irqrestore);
 396
 397 void __lockfunc _write_unlock_irq(rwlock_t *lock)
 398 {
 399         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 400         _raw_write_unlock(lock);
 401         local_irq_enable();
 402         preempt_enable();
 403 }
 404 EXPORT_SYMBOL(_write_unlock_irq);
 405
 406 void __lockfunc _write_unlock_bh(rwlock_t *lock)
 407 {
 408         rwlock_release(&lock->dep_map, 1, _RET_IP_);
 409         _raw_write_unlock(lock);
 410         preempt_enable_no_resched();
 411         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 412 }
 413 EXPORT_SYMBOL(_write_unlock_bh);
 414
 415 int __lockfunc _spin_trylock_bh(spinlock_t *lock)
 416 {
 417         local_bh_disable();
 418         preempt_disable();
 419         if (_raw_spin_trylock(lock)) {
 420                 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
 421                 return 1;
 422         }
 423
 424         preempt_enable_no_resched();
 425         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 426         return 0;
 427 }
 428 EXPORT_SYMBOL(_spin_trylock_bh);
 429
 430 int in_lock_functions(unsigned long addr)
 431 {
 432         /* Linker adds these: start and end of __lockfunc functions */
 433         extern char __lock_text_start[], __lock_text_end[];
 434
 435         return addr >= (unsigned long)__lock_text_start
 436         && addr < (unsigned long)__lock_text_end;
 437 }
 438 EXPORT_SYMBOL(in_lock_functions);