X-Git-Url: https://git.openpandora.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=include%2Fasm-mips%2Fatomic.h;h=1ac50b6c47adfc634ad55a3b023dc1d63fce7d3e;hb=03feb0524660bcd890674d11d29a1873ca14d13c;hp=e64abc0d82217068d4e73fce05a31afe5009e9da;hpb=ae3e0218621db0590163b2d5c424ef1f340e3cc6;p=pandora-kernel.git diff --git a/include/asm-mips/atomic.h b/include/asm-mips/atomic.h index e64abc0d8221..1ac50b6c47ad 100644 --- a/include/asm-mips/atomic.h +++ b/include/asm-mips/atomic.h @@ -9,20 +9,13 @@ * License. See the file "COPYING" in the main directory of this archive * for more details. * - * Copyright (C) 1996, 97, 99, 2000, 03, 04 by Ralf Baechle + * Copyright (C) 1996, 97, 99, 2000, 03, 04, 06 by Ralf Baechle */ - -/* - * As workaround for the ATOMIC_DEC_AND_LOCK / atomic_dec_and_lock mess in - * we have to include outside the - * main big wrapper ... - */ -#include - #ifndef _ASM_ATOMIC_H #define _ASM_ATOMIC_H #include +#include #include #include @@ -76,16 +69,19 @@ static __inline__ void atomic_add(int i, atomic_t * v) "1: ll %0, %1 # atomic_add \n" " addu %0, %2 \n" " sc %0, %1 \n" - " beqz %0, 1b \n" + " beqz %0, 2f \n" + " .subsection 2 \n" + "2: b 1b \n" + " .previous \n" " .set mips0 \n" : "=&r" (temp), "=m" (v->counter) : "Ir" (i), "m" (v->counter)); } else { unsigned long flags; - local_irq_save(flags); + raw_local_irq_save(flags); v->counter += i; - local_irq_restore(flags); + raw_local_irq_restore(flags); } } @@ -118,16 +114,19 @@ static __inline__ void atomic_sub(int i, atomic_t * v) "1: ll %0, %1 # atomic_sub \n" " subu %0, %2 \n" " sc %0, %1 \n" - " beqz %0, 1b \n" + " beqz %0, 2f \n" + " .subsection 2 \n" + "2: b 1b \n" + " .previous \n" " .set mips0 \n" : "=&r" (temp), "=m" (v->counter) : "Ir" (i), "m" (v->counter)); } else { unsigned long flags; - local_irq_save(flags); + raw_local_irq_save(flags); v->counter -= i; - local_irq_restore(flags); + raw_local_irq_restore(flags); } } @@ -138,6 +137,8 @@ static __inline__ int atomic_add_return(int i, atomic_t * v) { unsigned long result; + smp_mb(); + if (cpu_has_llsc && R10000_LLSC_WAR) { unsigned long temp; @@ -148,7 +149,6 @@ static __inline__ int atomic_add_return(int i, atomic_t * v) " sc %0, %2 \n" " beqzl %0, 1b \n" " addu %0, %1, %3 \n" - " sync \n" " .set mips0 \n" : "=&r" (result), "=&r" (temp), "=m" (v->counter) : "Ir" (i), "m" (v->counter) @@ -161,9 +161,11 @@ static __inline__ int atomic_add_return(int i, atomic_t * v) "1: ll %1, %2 # atomic_add_return \n" " addu %0, %1, %3 \n" " sc %0, %2 \n" - " beqz %0, 1b \n" + " beqz %0, 2f \n" " addu %0, %1, %3 \n" - " sync \n" + " .subsection 2 \n" + "2: b 1b \n" + " .previous \n" " .set mips0 \n" : "=&r" (result), "=&r" (temp), "=m" (v->counter) : "Ir" (i), "m" (v->counter) @@ -171,13 +173,15 @@ static __inline__ int atomic_add_return(int i, atomic_t * v) } else { unsigned long flags; - local_irq_save(flags); + raw_local_irq_save(flags); result = v->counter; result += i; v->counter = result; - local_irq_restore(flags); + raw_local_irq_restore(flags); } + smp_mb(); + return result; } @@ -185,6 +189,8 @@ static __inline__ int atomic_sub_return(int i, atomic_t * v) { unsigned long result; + smp_mb(); + if (cpu_has_llsc && R10000_LLSC_WAR) { unsigned long temp; @@ -195,7 +201,6 @@ static __inline__ int atomic_sub_return(int i, atomic_t * v) " sc %0, %2 \n" " beqzl %0, 1b \n" " subu %0, %1, %3 \n" - " sync \n" " .set mips0 \n" : "=&r" (result), "=&r" (temp), "=m" (v->counter) : "Ir" (i), "m" (v->counter) @@ -208,9 +213,11 @@ static __inline__ int atomic_sub_return(int i, atomic_t * v) "1: ll %1, %2 # atomic_sub_return \n" " subu %0, %1, %3 \n" " sc %0, %2 \n" - " beqz %0, 1b \n" + " beqz %0, 2f \n" " subu %0, %1, %3 \n" - " sync \n" + " .subsection 2 \n" + "2: b 1b \n" + " .previous \n" " .set mips0 \n" : "=&r" (result), "=&r" (temp), "=m" (v->counter) : "Ir" (i), "m" (v->counter) @@ -218,13 +225,15 @@ static __inline__ int atomic_sub_return(int i, atomic_t * v) } else { unsigned long flags; - local_irq_save(flags); + raw_local_irq_save(flags); result = v->counter; result -= i; v->counter = result; - local_irq_restore(flags); + raw_local_irq_restore(flags); } + smp_mb(); + return result; } @@ -240,6 +249,8 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v) { unsigned long result; + smp_mb(); + if (cpu_has_llsc && R10000_LLSC_WAR) { unsigned long temp; @@ -253,7 +264,6 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v) " beqzl %0, 1b \n" " subu %0, %1, %3 \n" " .set reorder \n" - " sync \n" "1: \n" " .set mips0 \n" : "=&r" (result), "=&r" (temp), "=m" (v->counter) @@ -269,11 +279,13 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v) " bltz %0, 1f \n" " sc %0, %2 \n" " .set noreorder \n" - " beqz %0, 1b \n" + " beqz %0, 2f \n" " subu %0, %1, %3 \n" " .set reorder \n" - " sync \n" "1: \n" + " .subsection 2 \n" + "2: b 1b \n" + " .previous \n" " .set mips0 \n" : "=&r" (result), "=&r" (temp), "=m" (v->counter) : "Ir" (i), "m" (v->counter) @@ -281,14 +293,16 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v) } else { unsigned long flags; - local_irq_save(flags); + raw_local_irq_save(flags); result = v->counter; result -= i; if (result >= 0) v->counter = result; - local_irq_restore(flags); + raw_local_irq_restore(flags); } + smp_mb(); + return result; } @@ -383,7 +397,7 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v) #ifdef CONFIG_64BIT -typedef struct { volatile __s64 counter; } atomic64_t; +typedef struct { volatile long counter; } atomic64_t; #define ATOMIC64_INIT(i) { (i) } @@ -430,16 +444,19 @@ static __inline__ void atomic64_add(long i, atomic64_t * v) "1: lld %0, %1 # atomic64_add \n" " addu %0, %2 \n" " scd %0, %1 \n" - " beqz %0, 1b \n" + " beqz %0, 2f \n" + " .subsection 2 \n" + "2: b 1b \n" + " .previous \n" " .set mips0 \n" : "=&r" (temp), "=m" (v->counter) : "Ir" (i), "m" (v->counter)); } else { unsigned long flags; - local_irq_save(flags); + raw_local_irq_save(flags); v->counter += i; - local_irq_restore(flags); + raw_local_irq_restore(flags); } } @@ -472,16 +489,19 @@ static __inline__ void atomic64_sub(long i, atomic64_t * v) "1: lld %0, %1 # atomic64_sub \n" " subu %0, %2 \n" " scd %0, %1 \n" - " beqz %0, 1b \n" + " beqz %0, 2f \n" + " .subsection 2 \n" + "2: b 1b \n" + " .previous \n" " .set mips0 \n" : "=&r" (temp), "=m" (v->counter) : "Ir" (i), "m" (v->counter)); } else { unsigned long flags; - local_irq_save(flags); + raw_local_irq_save(flags); v->counter -= i; - local_irq_restore(flags); + raw_local_irq_restore(flags); } } @@ -492,6 +512,8 @@ static __inline__ long atomic64_add_return(long i, atomic64_t * v) { unsigned long result; + smp_mb(); + if (cpu_has_llsc && R10000_LLSC_WAR) { unsigned long temp; @@ -502,7 +524,6 @@ static __inline__ long atomic64_add_return(long i, atomic64_t * v) " scd %0, %2 \n" " beqzl %0, 1b \n" " addu %0, %1, %3 \n" - " sync \n" " .set mips0 \n" : "=&r" (result), "=&r" (temp), "=m" (v->counter) : "Ir" (i), "m" (v->counter) @@ -515,9 +536,11 @@ static __inline__ long atomic64_add_return(long i, atomic64_t * v) "1: lld %1, %2 # atomic64_add_return \n" " addu %0, %1, %3 \n" " scd %0, %2 \n" - " beqz %0, 1b \n" + " beqz %0, 2f \n" " addu %0, %1, %3 \n" - " sync \n" + " .subsection 2 \n" + "2: b 1b \n" + " .previous \n" " .set mips0 \n" : "=&r" (result), "=&r" (temp), "=m" (v->counter) : "Ir" (i), "m" (v->counter) @@ -525,13 +548,15 @@ static __inline__ long atomic64_add_return(long i, atomic64_t * v) } else { unsigned long flags; - local_irq_save(flags); + raw_local_irq_save(flags); result = v->counter; result += i; v->counter = result; - local_irq_restore(flags); + raw_local_irq_restore(flags); } + smp_mb(); + return result; } @@ -539,6 +564,8 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v) { unsigned long result; + smp_mb(); + if (cpu_has_llsc && R10000_LLSC_WAR) { unsigned long temp; @@ -549,7 +576,6 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v) " scd %0, %2 \n" " beqzl %0, 1b \n" " subu %0, %1, %3 \n" - " sync \n" " .set mips0 \n" : "=&r" (result), "=&r" (temp), "=m" (v->counter) : "Ir" (i), "m" (v->counter) @@ -562,9 +588,11 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v) "1: lld %1, %2 # atomic64_sub_return \n" " subu %0, %1, %3 \n" " scd %0, %2 \n" - " beqz %0, 1b \n" + " beqz %0, 2f \n" " subu %0, %1, %3 \n" - " sync \n" + " .subsection 2 \n" + "2: b 1b \n" + " .previous \n" " .set mips0 \n" : "=&r" (result), "=&r" (temp), "=m" (v->counter) : "Ir" (i), "m" (v->counter) @@ -572,13 +600,15 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v) } else { unsigned long flags; - local_irq_save(flags); + raw_local_irq_save(flags); result = v->counter; result -= i; v->counter = result; - local_irq_restore(flags); + raw_local_irq_restore(flags); } + smp_mb(); + return result; } @@ -594,6 +624,8 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v) { unsigned long result; + smp_mb(); + if (cpu_has_llsc && R10000_LLSC_WAR) { unsigned long temp; @@ -607,7 +639,6 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v) " beqzl %0, 1b \n" " dsubu %0, %1, %3 \n" " .set reorder \n" - " sync \n" "1: \n" " .set mips0 \n" : "=&r" (result), "=&r" (temp), "=m" (v->counter) @@ -623,11 +654,13 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v) " bltz %0, 1f \n" " scd %0, %2 \n" " .set noreorder \n" - " beqz %0, 1b \n" + " beqz %0, 2f \n" " dsubu %0, %1, %3 \n" " .set reorder \n" - " sync \n" "1: \n" + " .subsection 2 \n" + "2: b 1b \n" + " .previous \n" " .set mips0 \n" : "=&r" (result), "=&r" (temp), "=m" (v->counter) : "Ir" (i), "m" (v->counter) @@ -635,14 +668,16 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v) } else { unsigned long flags; - local_irq_save(flags); + raw_local_irq_save(flags); result = v->counter; result -= i; if (result >= 0) v->counter = result; - local_irq_restore(flags); + raw_local_irq_restore(flags); } + smp_mb(); + return result; }