From: Zwane Mwaikambo <zwane@linuxpower.ca>
Following up on Keith's code, I adapted the i386 code to allow enabling
interrupts during contested locks depending on previous interrupt
enable status. Obviously there will be a text increase (only for non
CONFIG_SPINLINE case), although it doesn't seem so bad, there will be an
increased exit latency when we attempt a lock acquisition after spinning
due to the extra instructions. How much this will affect performance I'm
not sure yet as I haven't had time to micro bench.
text data bss dec hex filename
2628024 921731 0 3549755 362a3b vmlinux-after
2621369 921731 0 3543100 36103c vmlinux-before
2618313 919222 0 3537535 35fa7f vmlinux-spinline
The code has been stress tested on a 16x NUMAQ (courtesy OSDL).
Signed-off-by: Andrew Morton <akpm@osdl.org>
---
25-akpm/include/asm-i386/spinlock.h | 32 +++++++++++++++++++++++++++++++-
1 files changed, 31 insertions(+), 1 deletion(-)
diff -puN include/asm-i386/spinlock.h~allow-i386-to-reenable-interrupts-on-lock-contention include/asm-i386/spinlock.h
--- 25/include/asm-i386/spinlock.h~allow-i386-to-reenable-interrupts-on-lock-contention 2004-06-20 18:58:46.252573992 -0700
+++ 25-akpm/include/asm-i386/spinlock.h 2004-06-20 18:58:46.256573384 -0700
@@ -42,7 +42,6 @@ typedef struct {
#define spin_is_locked(x) (*(volatile signed char *)(&(x)->lock) <= 0)
#define spin_unlock_wait(x) do { barrier(); } while(spin_is_locked(x))
-#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
#define spin_lock_string \
"\n1:\t" \
@@ -56,6 +55,23 @@ typedef struct {
"jmp 1b\n" \
LOCK_SECTION_END
+#define spin_lock_string_flags \
+ "\n1:\t" \
+ "lock ; decb %0\n\t" \
+ "js 2f\n\t" \
+ LOCK_SECTION_START("") \
+ "2:\t" \
+ "testl $0x200, %1\n\t" \
+ "jz 3f\n\t" \
+ "sti\n\t" \
+ "3:\t" \
+ "rep;nop\n\t" \
+ "cmpb $0, %0\n\t" \
+ "jle 3b\n\t" \
+ "cli\n\t" \
+ "jmp 1b\n" \
+ LOCK_SECTION_END
+
/*
* This works. Despite all the confusion.
* (except on PPro SMP or if we are using OOSTORE)
@@ -126,6 +142,20 @@ here:
:"=m" (lock->lock) : : "memory");
}
+static inline void _raw_spin_lock_flags (spinlock_t *lock, unsigned long flags)
+{
+#ifdef CONFIG_DEBUG_SPINLOCK
+ __label__ here;
+here:
+ if (unlikely(lock->magic != SPINLOCK_MAGIC)) {
+ printk("eip: %p\n", &&here);
+ BUG();
+ }
+#endif
+ __asm__ __volatile__(
+ spin_lock_string_flags
+ :"=m" (lock->lock) : "r" (flags) : "memory");
+}
/*
* Read-write spinlocks, allowing multiple readers
_