Merge branch 'release' of git://git.kernel.org/pub/scm/linux/kernel/git/lenb/linux...

[pandora-kernel.git] / include / asm-i386 / spinlock.h

diff --git a/include/asm-i386/spinlock.h b/include/asm-i386/spinlock.h
index 2360435..87c40f8 100644 (file)
--- a/include/asm-i386/spinlock.h
+++ b/include/asm-i386/spinlock.h
@@ -4,7 +4,6 @@
 #include <asm/atomic.h>
 #include <asm/rwlock.h>
 #include <asm/page.h>
-#include <linux/config.h>
 #include <linux/compiler.h>
 
 /*
@@ -32,35 +31,57 @@
        "jmp 1b\n" \
        "3:\n\t"
 
+/*
+ * NOTE: there's an irqs-on section here, which normally would have to be
+ * irq-traced, but on CONFIG_TRACE_IRQFLAGS we never use
+ * __raw_spin_lock_string_flags().
+ */
 #define __raw_spin_lock_string_flags \
        "\n1:\t" \
        "lock ; decb %0\n\t" \
-       "jns 4f\n\t" \
+       "jns 5f\n" \
        "2:\t" \
        "testl $0x200, %1\n\t" \
-       "jz 3f\n\t" \
-       "sti\n\t" \
+       "jz 4f\n\t" \
+       "sti\n" \
        "3:\t" \
        "rep;nop\n\t" \
        "cmpb $0, %0\n\t" \
        "jle 3b\n\t" \
        "cli\n\t" \
        "jmp 1b\n" \
-       "4:\n\t"
+       "4:\t" \
+       "rep;nop\n\t" \
+       "cmpb $0, %0\n\t" \
+       "jg 1b\n\t" \
+       "jmp 4b\n" \
+       "5:\n\t"
+
+#define __raw_spin_lock_string_up \
+       "\n\tdecb %0"
 
 static inline void __raw_spin_lock(raw_spinlock_t *lock)
 {
-       __asm__ __volatile__(
-               __raw_spin_lock_string
-               :"=m" (lock->slock) : : "memory");
+       alternative_smp(
+               __raw_spin_lock_string,
+               __raw_spin_lock_string_up,
+               "=m" (lock->slock) : : "memory");
 }
 
+/*
+ * It is easier for the lock validator if interrupts are not re-enabled
+ * in the middle of a lock-acquire. This is a performance feature anyway
+ * so we turn it off:
+ */
+#ifndef CONFIG_PROVE_LOCKING
 static inline void __raw_spin_lock_flags(raw_spinlock_t *lock, unsigned long flags)
 {
-       __asm__ __volatile__(
-               __raw_spin_lock_string_flags
-               :"=m" (lock->slock) : "r" (flags) : "memory");
+       alternative_smp(
+               __raw_spin_lock_string_flags,
+               __raw_spin_lock_string_up,
+               "=m" (lock->slock) : "r" (flags) : "memory");
 }
+#endif
 
 static inline int __raw_spin_trylock(raw_spinlock_t *lock)
 {
@@ -178,12 +199,12 @@ static inline int __raw_write_trylock(raw_rwlock_t *lock)
 
 static inline void __raw_read_unlock(raw_rwlock_t *rw)
 {
-       asm volatile("lock ; incl %0" :"=m" (rw->lock) : : "memory");
+       asm volatile(LOCK_PREFIX "incl %0" :"=m" (rw->lock) : : "memory");
 }
 
 static inline void __raw_write_unlock(raw_rwlock_t *rw)
 {
-       asm volatile("lock ; addl $" RW_LOCK_BIAS_STR ", %0"
+       asm volatile(LOCK_PREFIX "addl $" RW_LOCK_BIAS_STR ", %0"
                                 : "=m" (rw->lock) : : "memory");
 }