From: Keith Owens <kaos@sgi.com>
As requested by Linus, update all architectures to add the common
infrastructure. Tested on ia64 and i386.
Enable interrupts while waiting for a disabled spinlock, but only if
interrupts were enabled before issuing spin_lock_irqsave().
This patch consists of three sections :-
* An architecture independent change to call _raw_spin_lock_flags()
instead of _raw_spin_lock() when the flags are available.
* An ia64 specific change to implement _raw_spin_lock_flags() and to
define _raw_spin_lock(lock) as _raw_spin_lock_flags(lock, 0) for the
ASM_SUPPORTED case.
* Patches for all other architectures and for ia64 with !ASM_SUPPORTED
to map _raw_spin_lock_flags(lock, flags) to _raw_spin_lock(lock).
Architecture maintainers can define _raw_spin_lock_flags() to do
something useful if they want to enable interrupts while waiting for
a disabled spinlock.
---
25-akpm/arch/ia64/kernel/head.S | 22 +++++++++++++++++-----
25-akpm/include/asm-alpha/spinlock.h | 1 +
25-akpm/include/asm-arm/spinlock.h | 1 +
25-akpm/include/asm-i386/spinlock.h | 1 +
25-akpm/include/asm-ia64/spinlock.h | 24 +++++++++++++++---------
25-akpm/include/asm-mips/spinlock.h | 1 +
25-akpm/include/asm-parisc/spinlock.h | 1 +
25-akpm/include/asm-ppc/spinlock.h | 1 +
25-akpm/include/asm-ppc64/spinlock.h | 1 +
25-akpm/include/asm-s390/spinlock.h | 1 +
25-akpm/include/asm-sh/spinlock.h | 1 +
25-akpm/include/asm-sparc/spinlock.h | 2 ++
25-akpm/include/asm-sparc64/spinlock.h | 2 ++
25-akpm/include/asm-x86_64/spinlock.h | 1 +
25-akpm/include/linux/spinlock.h | 2 +-
15 files changed, 47 insertions(+), 15 deletions(-)
diff -puN arch/ia64/kernel/head.S~allow-architectures-to-reenable-interrupts-on-contended-spinlocks arch/ia64/kernel/head.S
--- 25/arch/ia64/kernel/head.S~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.678377240 -0700
+++ 25-akpm/arch/ia64/kernel/head.S 2004-04-27 21:19:55.702373592 -0700
@@ -866,12 +866,14 @@ SET_REG(b5);
* Inputs:
* ar.pfs - saved CFM of caller
* ar.ccv - 0 (and available for use)
+ * r27 - flags from spin_lock_irqsave or 0. Must be preserved.
* r28 - available for use.
* r29 - available for use.
* r30 - available for use.
* r31 - address of lock, available for use.
* b6 - return address
* p14 - available for use.
+ * p15 - used to track flag status.
*
* If you patch this code to use more registers, do not forget to update
* the clobber lists for spin_lock() in include/asm-ia64/spinlock.h.
@@ -885,22 +887,26 @@ GLOBAL_ENTRY(ia64_spinlock_contention_pr
.save rp, r28
.body
nop 0
- nop 0
+ tbit.nz p15,p0=r27,IA64_PSR_I_BIT
.restore sp // pop existing prologue after next insn
mov b6 = r28
.prologue
.save ar.pfs, r0
.altrp b6
.body
+ ;;
+(p15) ssm psr.i // reenable interrupts if they were on
+ // DavidM says that srlz.d is slow and is not required in this case
.wait:
// exponential backoff, kdb, lockmeter etc. go in here
hint @pause
ld4 r30=[r31] // don't use ld4.bias; if it's contended, we won't write the word
nop 0
;;
- cmp4.eq p14,p0=r30,r0
-(p14) br.cond.sptk.few b6 // lock is now free, try to acquire
- br.cond.sptk.few .wait
+ cmp4.ne p14,p0=r30,r0
+(p14) br.cond.sptk.few .wait
+(p15) rsm psr.i // disable interrupts if we reenabled them
+ br.cond.sptk.few b6 // lock is now free, try to acquire
END(ia64_spinlock_contention_pre3_4)
#else
@@ -909,14 +915,20 @@ GLOBAL_ENTRY(ia64_spinlock_contention)
.prologue
.altrp b6
.body
+ tbit.nz p15,p0=r27,IA64_PSR_I_BIT
+ ;;
.wait:
+(p15) ssm psr.i // reenable interrupts if they were on
+ // DavidM says that srlz.d is slow and is not required in this case
+.wait2:
// exponential backoff, kdb, lockmeter etc. go in here
hint @pause
ld4 r30=[r31] // don't use ld4.bias; if it's contended, we won't write the word
;;
cmp4.ne p14,p0=r30,r0
mov r30 = 1
-(p14) br.cond.sptk.few .wait
+(p14) br.cond.sptk.few .wait2
+(p15) rsm psr.i // disable interrupts if we reenabled them
;;
cmpxchg4.acq r30=[r31], r30, ar.ccv
;;
diff -puN include/asm-alpha/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-alpha/spinlock.h
--- 25/include/asm-alpha/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.680376936 -0700
+++ 25-akpm/include/asm-alpha/spinlock.h 2004-04-27 21:19:55.703373440 -0700
@@ -40,6 +40,7 @@ typedef struct {
#define spin_is_locked(x) ((x)->lock != 0)
#define spin_unlock_wait(x) ({ do { barrier(); } while ((x)->lock); })
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
#ifdef CONFIG_DEBUG_SPINLOCK
extern void _raw_spin_unlock(spinlock_t * lock);
diff -puN include/asm-arm/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-arm/spinlock.h
--- 25/include/asm-arm/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.681376784 -0700
+++ 25-akpm/include/asm-arm/spinlock.h 2004-04-27 21:19:55.703373440 -0700
@@ -24,6 +24,7 @@ typedef struct {
#define spin_lock_init(x) do { *(x) = SPIN_LOCK_UNLOCKED; } while (0)
#define spin_is_locked(x) ((x)->lock != 0)
#define spin_unlock_wait(x) do { barrier(); } while (spin_is_locked(x))
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
static inline void _raw_spin_lock(spinlock_t *lock)
{
diff -puN include/asm-i386/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-i386/spinlock.h
--- 25/include/asm-i386/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.682376632 -0700
+++ 25-akpm/include/asm-i386/spinlock.h 2004-04-27 21:19:55.704373288 -0700
@@ -42,6 +42,7 @@ typedef struct {
#define spin_is_locked(x) (*(volatile signed char *)(&(x)->lock) <= 0)
#define spin_unlock_wait(x) do { barrier(); } while(spin_is_locked(x))
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
#ifdef CONFIG_SPINLINE
diff -puN include/asm-ia64/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-ia64/spinlock.h
--- 25/include/asm-ia64/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.684376328 -0700
+++ 25-akpm/include/asm-ia64/spinlock.h 2004-04-27 21:19:55.701373744 -0700
@@ -32,10 +32,10 @@ typedef struct {
* carefully coded to touch only those registers that spin_lock() marks "clobbered".
*/
-#define IA64_SPINLOCK_CLOBBERS "ar.ccv", "ar.pfs", "p14", "r28", "r29", "r30", "b6", "memory"
+#define IA64_SPINLOCK_CLOBBERS "ar.ccv", "ar.pfs", "p14", "p15", "r27", "r28", "r29", "r30", "b6", "memory"
static inline void
-_raw_spin_lock (spinlock_t *lock)
+_raw_spin_lock_flags (spinlock_t *lock, unsigned long flags)
{
register volatile unsigned int *ptr asm ("r31") = &lock->lock;
@@ -50,9 +50,10 @@ _raw_spin_lock (spinlock_t *lock)
"cmpxchg4.acq r30 = [%1], r30, ar.ccv\n\t"
"movl r29 = ia64_spinlock_contention_pre3_4;;\n\t"
"cmp4.ne p14, p0 = r30, r0\n\t"
- "mov b6 = r29;;\n"
+ "mov b6 = r29;;\n\t"
+ "mov r27=%2\n\t"
"(p14) br.cond.spnt.many b6"
- : "=r"(ptr) : "r"(ptr) : IA64_SPINLOCK_CLOBBERS);
+ : "=r"(ptr) : "r"(ptr), "r" (flags) : IA64_SPINLOCK_CLOBBERS);
# else
asm volatile ("{\n\t"
" mov ar.ccv = r0\n\t"
@@ -60,33 +61,38 @@ _raw_spin_lock (spinlock_t *lock)
" mov r30 = 1;;\n\t"
"}\n\t"
"cmpxchg4.acq r30 = [%1], r30, ar.ccv;;\n\t"
- "cmp4.ne p14, p0 = r30, r0\n"
+ "cmp4.ne p14, p0 = r30, r0\n\t"
+ "mov r27=%2\n\t"
"(p14) brl.cond.spnt.many ia64_spinlock_contention_pre3_4;;"
- : "=r"(ptr) : "r"(ptr) : IA64_SPINLOCK_CLOBBERS);
+ : "=r"(ptr) : "r"(ptr), "r" (flags) : IA64_SPINLOCK_CLOBBERS);
# endif /* CONFIG_MCKINLEY */
#else
# ifdef CONFIG_ITANIUM
/* don't use brl on Itanium... */
/* mis-declare, so we get the entry-point, not it's function descriptor: */
asm volatile ("mov r30 = 1\n\t"
+ "mov r27=%2\n\t"
"mov ar.ccv = r0;;\n\t"
"cmpxchg4.acq r30 = [%0], r30, ar.ccv\n\t"
"movl r29 = ia64_spinlock_contention;;\n\t"
"cmp4.ne p14, p0 = r30, r0\n\t"
- "mov b6 = r29;;\n"
+ "mov b6 = r29;;\n\t"
"(p14) br.call.spnt.many b6 = b6"
- : "=r"(ptr) : "r"(ptr) : IA64_SPINLOCK_CLOBBERS);
+ : "=r"(ptr) : "r"(ptr), "r" (flags) : IA64_SPINLOCK_CLOBBERS);
# else
asm volatile ("mov r30 = 1\n\t"
+ "mov r27=%2\n\t"
"mov ar.ccv = r0;;\n\t"
"cmpxchg4.acq r30 = [%0], r30, ar.ccv;;\n\t"
"cmp4.ne p14, p0 = r30, r0\n\t"
"(p14) brl.call.spnt.many b6=ia64_spinlock_contention;;"
- : "=r"(ptr) : "r"(ptr) : IA64_SPINLOCK_CLOBBERS);
+ : "=r"(ptr) : "r"(ptr), "r" (flags) : IA64_SPINLOCK_CLOBBERS);
# endif /* CONFIG_MCKINLEY */
#endif
}
+#define _raw_spin_lock(lock) _raw_spin_lock_flags(lock, 0)
#else /* !ASM_SUPPORTED */
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
# define _raw_spin_lock(x) \
do { \
__u32 *ia64_spinlock_ptr = (__u32 *) (x); \
diff -puN include/asm-mips/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-mips/spinlock.h
--- 25/include/asm-mips/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.685376176 -0700
+++ 25-akpm/include/asm-mips/spinlock.h 2004-04-27 21:19:55.704373288 -0700
@@ -23,6 +23,7 @@ typedef struct {
#define spin_is_locked(x) ((x)->lock != 0)
#define spin_unlock_wait(x) do { barrier(); } while ((x)->lock)
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
/*
* Simple spin lock operations. There are two variants, one clears IRQ's
diff -puN include/asm-parisc/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-parisc/spinlock.h
--- 25/include/asm-parisc/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.687375872 -0700
+++ 25-akpm/include/asm-parisc/spinlock.h 2004-04-27 21:19:55.705373136 -0700
@@ -15,6 +15,7 @@
#define spin_is_locked(x) ((x)->lock == 0)
#define spin_unlock_wait(x) do { barrier(); } while(((volatile spinlock_t *)(x))->lock == 0)
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
#if 1
#define _raw_spin_lock(x) do { \
diff -puN include/asm-ppc64/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-ppc64/spinlock.h
--- 25/include/asm-ppc64/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.688375720 -0700
+++ 25-akpm/include/asm-ppc64/spinlock.h 2004-04-27 21:19:55.705373136 -0700
@@ -22,6 +22,7 @@ typedef struct {
#define SPIN_LOCK_UNLOCKED (spinlock_t) { 0 }
#define spin_is_locked(x) ((x)->lock != 0)
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
static __inline__ int _raw_spin_trylock(spinlock_t *lock)
{
diff -puN include/asm-ppc/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-ppc/spinlock.h
--- 25/include/asm-ppc/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.689375568 -0700
+++ 25-akpm/include/asm-ppc/spinlock.h 2004-04-27 21:19:55.705373136 -0700
@@ -27,6 +27,7 @@ typedef struct {
#define spin_lock_init(x) do { *(x) = SPIN_LOCK_UNLOCKED; } while(0)
#define spin_is_locked(x) ((x)->lock != 0)
#define spin_unlock_wait(x) do { barrier(); } while(spin_is_locked(x))
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
#ifndef CONFIG_DEBUG_SPINLOCK
diff -puN include/asm-s390/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-s390/spinlock.h
--- 25/include/asm-s390/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.690375416 -0700
+++ 25-akpm/include/asm-s390/spinlock.h 2004-04-27 21:19:55.706372984 -0700
@@ -42,6 +42,7 @@ typedef struct {
#define spin_lock_init(lp) do { (lp)->lock = 0; } while(0)
#define spin_unlock_wait(lp) do { barrier(); } while(((volatile spinlock_t *)(lp))->lock)
#define spin_is_locked(x) ((x)->lock != 0)
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
extern inline void _raw_spin_lock(spinlock_t *lp)
{
diff -puN include/asm-sh/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-sh/spinlock.h
--- 25/include/asm-sh/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.691375264 -0700
+++ 25-akpm/include/asm-sh/spinlock.h 2004-04-27 21:19:55.706372984 -0700
@@ -25,6 +25,7 @@ typedef struct {
#define spin_is_locked(x) ((x)->lock != 0)
#define spin_unlock_wait(x) do { barrier(); } while (spin_is_locked(x))
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
/*
* Simple spin lock operations. There are two variants, one clears IRQ's
diff -puN include/asm-sparc64/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-sparc64/spinlock.h
--- 25/include/asm-sparc64/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.693374960 -0700
+++ 25-akpm/include/asm-sparc64/spinlock.h 2004-04-27 21:19:55.707372832 -0700
@@ -114,6 +114,8 @@ extern int _spin_trylock (spinlock_t *lo
#endif /* CONFIG_DEBUG_SPINLOCK */
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
+
/* Multi-reader locks, these are much saner than the 32-bit Sparc ones... */
#ifndef CONFIG_DEBUG_SPINLOCK
diff -puN include/asm-sparc/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-sparc/spinlock.h
--- 25/include/asm-sparc/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.694374808 -0700
+++ 25-akpm/include/asm-sparc/spinlock.h 2004-04-27 21:19:55.707372832 -0700
@@ -216,6 +216,8 @@ extern __inline__ void _raw_write_lock(r
#endif /* CONFIG_DEBUG_SPINLOCK */
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
+
#endif /* !(__ASSEMBLY__) */
#endif /* __SPARC_SPINLOCK_H */
diff -puN include/asm-x86_64/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-x86_64/spinlock.h
--- 25/include/asm-x86_64/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.695374656 -0700
+++ 25-akpm/include/asm-x86_64/spinlock.h 2004-04-27 21:19:55.708372680 -0700
@@ -41,6 +41,7 @@ typedef struct {
#define spin_is_locked(x) (*(volatile signed char *)(&(x)->lock) <= 0)
#define spin_unlock_wait(x) do { barrier(); } while(spin_is_locked(x))
+#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
#define spin_lock_string \
"\n1:\t" \
diff -puN include/linux/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/linux/spinlock.h
--- 25/include/linux/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.697374352 -0700
+++ 25-akpm/include/linux/spinlock.h 2004-04-27 21:19:55.701373744 -0700
@@ -280,7 +280,7 @@ do { \
do { \
local_irq_save(flags); \
preempt_disable(); \
- _raw_spin_lock(lock); \
+ _raw_spin_lock_flags(lock, flags); \
} while (0)
#define spin_lock_irq(lock) \
_