patch-2.1.34 linux/arch/sparc/mm/sun4c.c
Next file: linux/arch/sparc/prom/misc.c
Previous file: linux/arch/sparc/mm/srmmu.c
Back to the patch index
Back to the overall index
- Lines: 1752
- Date:
Fri Apr 11 10:47:35 1997
- Orig file:
v2.1.33/linux/arch/sparc/mm/sun4c.c
- Orig date:
Mon Mar 17 14:54:22 1997
diff -u --recursive --new-file v2.1.33/linux/arch/sparc/mm/sun4c.c linux/arch/sparc/mm/sun4c.c
@@ -1,4 +1,4 @@
-/* $Id: sun4c.c,v 1.139 1997/01/31 08:05:59 davem Exp $
+/* $Id: sun4c.c,v 1.143 1997/04/11 00:42:14 davem Exp $
* sun4c.c: Doing in software what should be done in hardware.
*
* Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu)
@@ -23,8 +23,48 @@
#include <asm/openprom.h>
#include <asm/mmu_context.h>
+/* TODO: Make it such that interrupt handlers cannot dick with
+ * the user segment lists, most of the cli/sti pairs can
+ * disappear once that is taken care of.
+ */
+
+/* XXX Ok the real performance win, I figure, will be to use a combined hashing
+ * XXX and bitmap scheme to keep track of what we have mapped where. The whole
+ * XXX incentive is to make it such that the range flushes can be serviced
+ * XXX always in near constant time. --DaveM
+ */
+
extern int num_segmaps, num_contexts;
+/* Define this to get extremely anal debugging, undefine for performance. */
+/* #define DEBUG_SUN4C_MM */
+
+#define UWINMASK_OFFSET (const unsigned long)(&(((struct task_struct *)0)->tss.uwinmask))
+
+/* This is used in many routines below. */
+#define FUW_INLINE do { \
+ register int ctr asm("g5"); \
+ ctr = 0; \
+ __asm__ __volatile__("\n" \
+ "1: ld [%%g6 + %2], %%g4 ! flush user windows\n" \
+ " orcc %%g0, %%g4, %%g0\n" \
+ " add %0, 1, %0\n" \
+ " bne 1b\n" \
+ " save %%sp, -64, %%sp\n" \
+ "2: subcc %0, 1, %0\n" \
+ " bne 2b\n" \
+ " restore %%g0, %%g0, %%g0\n" \
+ : "=&r" (ctr) \
+ : "0" (ctr), "i" (UWINMASK_OFFSET) \
+ : "g4", "cc"); \
+} while(0);
+
+/* That's it, we prom_halt() if the cache size is something other than 65536.
+ * So let's save some cycles and just use that everywhere except for that bootup
+ * sanity check.
+ */
+#define SUN4C_VAC_SIZE 65536
+
#define SUN4C_KERNEL_BUCKETS 32
#ifndef MAX
@@ -100,7 +140,7 @@
/* Clear 'valid' bit in all cache line tags */
begin = AC_CACHETAGS;
- end = (AC_CACHETAGS + sun4c_vacinfo.num_bytes);
+ end = (AC_CACHETAGS + SUN4C_VAC_SIZE);
while(begin < end) {
__asm__ __volatile__("sta %%g0, [%0] %1\n\t" : :
"r" (begin), "i" (ASI_CONTROL));
@@ -108,83 +148,68 @@
}
}
-/* Blow the entire current context out of the virtual cache. */
-static inline void sun4c_flush_context(void)
+/* Context level flush. */
+static inline void sun4c_flush_context_hw(void)
{
- extern unsigned long bcopy;
- unsigned long begin, end, flags;
+ unsigned long end = SUN4C_VAC_SIZE;
+ unsigned pgsz = PAGE_SIZE;
ctxflushes++;
+ __asm__ __volatile__("
+1: subcc %0, %2, %0
+ bg 1b
+ sta %%g0, [%0] %3
+ nop; nop; nop; ! Weitek hwbug
+" : "=&r" (end)
+ : "0" (end), "r" (pgsz), "i" (ASI_HWFLUSHCONTEXT)
+ : "cc");
+}
- begin = ((unsigned long) &bcopy);
- end = (begin + sun4c_vacinfo.num_bytes);
+/* Don't inline the software version as it eats too many cache lines if expanded. */
+static void sun4c_flush_context_sw(void)
+{
+ unsigned long nbytes = SUN4C_VAC_SIZE;
+ unsigned long lsize = sun4c_vacinfo.linesize;
- save_and_cli(flags);
- if(sun4c_vacinfo.linesize == 32) {
- while(begin < end) {
- __asm__ __volatile__("
- ld [%0 + 0x00], %%g0
- ld [%0 + 0x20], %%g0
- ld [%0 + 0x40], %%g0
- ld [%0 + 0x60], %%g0
- ld [%0 + 0x80], %%g0
- ld [%0 + 0xa0], %%g0
- ld [%0 + 0xc0], %%g0
- ld [%0 + 0xe0], %%g0
- ld [%0 + 0x100], %%g0
- ld [%0 + 0x120], %%g0
- ld [%0 + 0x140], %%g0
- ld [%0 + 0x160], %%g0
- ld [%0 + 0x180], %%g0
- ld [%0 + 0x1a0], %%g0
- ld [%0 + 0x1c0], %%g0
- ld [%0 + 0x1e0], %%g0
- " : : "r" (begin));
- begin += 512;
- }
- } else {
- while(begin < end) {
- __asm__ __volatile__("
- ld [%0 + 0x00], %%g0
- ld [%0 + 0x10], %%g0
- ld [%0 + 0x20], %%g0
- ld [%0 + 0x30], %%g0
- ld [%0 + 0x40], %%g0
- ld [%0 + 0x50], %%g0
- ld [%0 + 0x60], %%g0
- ld [%0 + 0x70], %%g0
- ld [%0 + 0x80], %%g0
- ld [%0 + 0x90], %%g0
- ld [%0 + 0xa0], %%g0
- ld [%0 + 0xb0], %%g0
- ld [%0 + 0xc0], %%g0
- ld [%0 + 0xd0], %%g0
- ld [%0 + 0xe0], %%g0
- ld [%0 + 0xf0], %%g0
- " : : "r" (begin));
- begin += 256;
- }
- }
- restore_flags(flags);
+ ctxflushes++;
+ __asm__ __volatile__("
+ add %2, %2, %%g1
+ add %2, %%g1, %%g2
+ add %2, %%g2, %%g3
+ add %2, %%g3, %%g4
+ add %2, %%g4, %%g5
+ add %2, %%g5, %%o4
+ add %2, %%o4, %%o5
+1: subcc %0, %%o5, %0
+ sta %%g0, [%0] %3
+ sta %%g0, [%0 + %2] %3
+ sta %%g0, [%0 + %%g1] %3
+ sta %%g0, [%0 + %%g2] %3
+ sta %%g0, [%0 + %%g3] %3
+ sta %%g0, [%0 + %%g4] %3
+ sta %%g0, [%0 + %%g5] %3
+ bg 1b
+ sta %%g0, [%1 + %%o4] %3
+" : "=&r" (nbytes)
+ : "0" (nbytes), "r" (lsize), "i" (ASI_FLUSHCTX)
+ : "g1", "g2", "g3", "g4", "g5", "o4", "o5", "cc");
}
/* Scrape the segment starting at ADDR from the virtual cache. */
static inline void sun4c_flush_segment(unsigned long addr)
{
- segflushes++;
- addr &= SUN4C_REAL_PGDIR_MASK;
-
if(sun4c_get_segmap(addr) == invalid_segment)
return;
+ segflushes++;
if(sun4c_vacinfo.do_hwflushes) {
- unsigned long end = (addr + sun4c_vacinfo.num_bytes);
+ unsigned long end = (addr + SUN4C_VAC_SIZE);
for( ; addr < end; addr += PAGE_SIZE)
- __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : :
+ __asm__ __volatile__("sta %%g0, [%0] %1;nop;nop;nop;\n\t" : :
"r" (addr), "i" (ASI_HWFLUSHSEG));
} else {
- unsigned long nbytes = sun4c_vacinfo.num_bytes;
+ unsigned long nbytes = SUN4C_VAC_SIZE;
unsigned long lsize = sun4c_vacinfo.linesize;
__asm__ __volatile__("add %2, %2, %%g1\n\t"
@@ -209,12 +234,61 @@
: "=&r" (addr), "=&r" (nbytes), "=&r" (lsize)
: "0" (addr), "1" (nbytes), "2" (lsize),
"i" (ASI_FLUSHSEG)
- : "g1", "g2", "g3", "g4", "g5", "o4", "o5");
+ : "g1", "g2", "g3", "g4", "g5", "o4", "o5", "cc");
+ }
+}
+
+/* Call this version when you know hardware flushes are available. */
+static inline void sun4c_flush_segment_hw(unsigned long addr)
+{
+ if(sun4c_get_segmap(addr) != invalid_segment) {
+ unsigned long end;
+
+ segflushes++;
+ for(end = addr + SUN4C_VAC_SIZE; addr < end; addr += PAGE_SIZE)
+ __asm__ __volatile__("sta %%g0, [%0] %1"
+ : : "r" (addr), "i" (ASI_HWFLUSHSEG));
+ /* Weitek POWER-UP hwbug workaround. */
+ __asm__ __volatile__("nop;nop;nop; ! Weitek hwbug");
+ }
+}
+
+/* Don't inline the software version as it eats too many cache lines if expanded. */
+static void sun4c_flush_segment_sw(unsigned long addr)
+{
+ if(sun4c_get_segmap(addr) != invalid_segment) {
+ unsigned long nbytes = SUN4C_VAC_SIZE;
+ unsigned long lsize = sun4c_vacinfo.linesize;
+
+ segflushes++;
+ __asm__ __volatile__("
+ add %2, %2, %%g1
+ add %2, %%g1, %%g2
+ add %2, %%g2, %%g3
+ add %2, %%g3, %%g4
+ add %2, %%g4, %%g5
+ add %2, %%g5, %%o4
+ add %2, %%o4, %%o5
+1: subcc %1, %%o5, %1
+ sta %%g0, [%0] %6
+ sta %%g0, [%0 + %2] %6
+ sta %%g0, [%0 + %%g1] %6
+ sta %%g0, [%0 + %%g2] %6
+ sta %%g0, [%0 + %%g3] %6
+ sta %%g0, [%0 + %%g4] %6
+ sta %%g0, [%0 + %%g5] %6
+ sta %%g0, [%0 + %%o4] %6
+ bg 1b
+ add %0, %%o5, %0
+" : "=&r" (addr), "=&r" (nbytes), "=&r" (lsize)
+ : "0" (addr), "1" (nbytes), "2" (lsize),
+ "i" (ASI_FLUSHSEG)
+ : "g1", "g2", "g3", "g4", "g5", "o4", "o5", "cc");
}
}
/* Bolix one page from the virtual cache. */
-static inline void sun4c_flush_page(unsigned long addr)
+static void sun4c_flush_page(unsigned long addr)
{
addr &= PAGE_MASK;
@@ -224,7 +298,7 @@
pageflushes++;
if(sun4c_vacinfo.do_hwflushes) {
- __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : :
+ __asm__ __volatile__("sta %%g0, [%0] %1;nop;nop;nop;\n\t" : :
"r" (addr), "i" (ASI_HWFLUSHPAGE));
} else {
unsigned long left = PAGE_SIZE;
@@ -252,7 +326,57 @@
: "=&r" (addr), "=&r" (left), "=&r" (lsize)
: "0" (addr), "1" (left), "2" (lsize),
"i" (ASI_FLUSHPG)
- : "g1", "g2", "g3", "g4", "g5", "o4", "o5");
+ : "g1", "g2", "g3", "g4", "g5", "o4", "o5", "cc");
+ }
+}
+
+/* Again, hw-only and sw-only cache page-level flush variants. */
+static inline void sun4c_flush_page_hw(unsigned long addr)
+{
+ addr &= PAGE_MASK;
+ if((sun4c_get_pte(addr) & (_SUN4C_PAGE_NOCACHE | _SUN4C_PAGE_VALID)) ==
+ _SUN4C_PAGE_VALID) {
+ pageflushes++;
+ __asm__ __volatile__("sta %%g0, [%0] %1"
+ : : "r" (addr), "i" (ASI_HWFLUSHPAGE));
+ /* Weitek POWER-UP hwbug workaround. */
+ __asm__ __volatile__("nop;nop;nop; ! Weitek hwbug");
+ }
+}
+
+/* Don't inline the software version as it eats too many cache lines if expanded. */
+static void sun4c_flush_page_sw(unsigned long addr)
+{
+ addr &= PAGE_MASK;
+ if((sun4c_get_pte(addr) & (_SUN4C_PAGE_NOCACHE | _SUN4C_PAGE_VALID)) ==
+ _SUN4C_PAGE_VALID) {
+ unsigned long left = PAGE_SIZE;
+ unsigned long lsize = sun4c_vacinfo.linesize;
+
+ pageflushes++;
+ __asm__ __volatile__("
+ add %2, %2, %%g1
+ add %2, %%g1, %%g2
+ add %2, %%g2, %%g3
+ add %2, %%g3, %%g4
+ add %2, %%g4, %%g5
+ add %2, %%g5, %%o4
+ add %2, %%o4, %%o5
+1: subcc %1, %%o5, %1
+ sta %%g0, [%0] %6
+ sta %%g0, [%0 + %2] %6
+ sta %%g0, [%0 + %%g1] %6
+ sta %%g0, [%0 + %%g2] %6
+ sta %%g0, [%0 + %%g3] %6
+ sta %%g0, [%0 + %%g4] %6
+ sta %%g0, [%0 + %%g5] %6
+ sta %%g0, [%0 + %%o4] %6
+ bg 1b
+ add %0, %%o5, %0
+" : "=&r" (addr), "=&r" (left), "=&r" (lsize)
+ : "0" (addr), "1" (left), "2" (lsize),
+ "i" (ASI_FLUSHPG)
+ : "g1", "g2", "g3", "g4", "g5", "o4", "o5", "cc");
}
}
@@ -615,26 +739,33 @@
ring->num_entries++;
}
-static inline void remove_ring(struct sun4c_mmu_ring *ring,
- struct sun4c_mmu_entry *entry)
+static inline void add_ring_ordered(struct sun4c_mmu_ring *ring,
+ struct sun4c_mmu_entry *entry)
{
- struct sun4c_mmu_entry *next = entry->next;
+ struct sun4c_mmu_entry *head = &ring->ringhd;
+ unsigned long addr = entry->vaddr;
- (next->prev = entry->prev)->next = next;
- ring->num_entries--;
+ if(head->next != &ring->ringhd) {
+ while((head->next != &ring->ringhd) && (head->next->vaddr < addr))
+ head = head->next;
+ }
+ entry->prev = head;
+ (entry->next = head->next)->prev = entry;
+ head->next = entry;
+ ring->num_entries++;
}
-static inline void recycle_ring(struct sun4c_mmu_ring *ring,
- struct sun4c_mmu_entry *entry)
+static inline void remove_ring(struct sun4c_mmu_ring *ring,
+ struct sun4c_mmu_entry *entry)
{
- struct sun4c_mmu_entry *head = &ring->ringhd;
struct sun4c_mmu_entry *next = entry->next;
(next->prev = entry->prev)->next = next;
- entry->prev = head;
- (entry->next = head->next)->prev = entry;
- head->next = entry;
- /* num_entries stays the same */
+ ring->num_entries--;
+#ifdef DEBUG_SUN4C_MM
+ if(ring->num_entries < 0)
+ panic("sun4c: Ring num_entries < 0!");
+#endif
}
static inline void free_user_entry(int ctx, struct sun4c_mmu_entry *entry)
@@ -646,7 +777,7 @@
static inline void assign_user_entry(int ctx, struct sun4c_mmu_entry *entry)
{
remove_ring(&sun4c_ufree_ring, entry);
- add_ring(sun4c_context_ring+ctx, entry);
+ add_ring_ordered(sun4c_context_ring+ctx, entry);
}
static inline void free_kernel_entry(struct sun4c_mmu_entry *entry,
@@ -724,24 +855,55 @@
}
}
-static inline void sun4c_demap_context(struct sun4c_mmu_ring *crp, unsigned char ctx)
+static void sun4c_demap_context_hw(struct sun4c_mmu_ring *crp, unsigned char ctx)
{
- struct sun4c_mmu_entry *this_entry, *next_entry;
+ struct sun4c_mmu_entry *head = &crp->ringhd;
unsigned long flags;
- int savectx = sun4c_get_context();
save_and_cli(flags);
- this_entry = crp->ringhd.next;
- flush_user_windows();
- sun4c_set_context(ctx);
- while(crp->num_entries) {
- next_entry = this_entry->next;
- sun4c_flush_segment(this_entry->vaddr);
- sun4c_user_unmap(this_entry);
- free_user_entry(ctx, this_entry);
- this_entry = next_entry;
+ if(head->next != head) {
+ struct sun4c_mmu_entry *entry = head->next;
+ int savectx = sun4c_get_context();
+
+ FUW_INLINE
+ sun4c_set_context(ctx);
+ sun4c_flush_context_hw();
+ do {
+ struct sun4c_mmu_entry *next = entry->next;
+
+ sun4c_user_unmap(entry);
+ free_user_entry(ctx, entry);
+
+ entry = next;
+ } while(entry != head);
+ sun4c_set_context(savectx);
+ }
+ restore_flags(flags);
+}
+
+static void sun4c_demap_context_sw(struct sun4c_mmu_ring *crp, unsigned char ctx)
+{
+ struct sun4c_mmu_entry *head = &crp->ringhd;
+ unsigned long flags;
+
+ save_and_cli(flags);
+ if(head->next != head) {
+ struct sun4c_mmu_entry *entry = head->next;
+ int savectx = sun4c_get_context();
+
+ FUW_INLINE
+ sun4c_set_context(ctx);
+ sun4c_flush_context_sw();
+ do {
+ struct sun4c_mmu_entry *next = entry->next;
+
+ sun4c_user_unmap(entry);
+ free_user_entry(ctx, entry);
+
+ entry = next;
+ } while(entry != head);
+ sun4c_set_context(savectx);
}
- sun4c_set_context(savectx);
restore_flags(flags);
}
@@ -749,17 +911,25 @@
{
/* by using .prev we get a kind of "lru" algorithm */
struct sun4c_mmu_entry *entry = crp->ringhd.prev;
+ unsigned long flags;
int savectx = sun4c_get_context();
- flush_user_windows();
+#ifdef DEBUG_SUN4C_MM
+ if(entry == &crp->ringhd)
+ panic("sun4c_demap_one: Freeing from empty ctx ring.");
+#endif
+ FUW_INLINE
+ save_and_cli(flags);
sun4c_set_context(ctx);
sun4c_flush_segment(entry->vaddr);
sun4c_user_unmap(entry);
free_user_entry(ctx, entry);
sun4c_set_context(savectx);
+ restore_flags(flags);
}
-static int sun4c_user_taken_entries = 0;
+static int sun4c_user_taken_entries = 0; /* This is how much we have. */
+static int max_user_taken_entries = 0; /* This limits us and prevents deadlock. */
static inline struct sun4c_mmu_entry *sun4c_kernel_strategy(void)
{
@@ -800,6 +970,10 @@
sun4c_kfree_ring.num_entries,
sun4c_kernel_ring.num_entries);
#endif
+#ifdef DEBUG_SUN4C_MM
+ if(sun4c_user_taken_entries < 0)
+ panic("sun4c_shrink_kernel_ring: taken < 0.");
+#endif
}
restore_flags(flags);
}
@@ -813,25 +987,50 @@
struct ctx_list *next_one;
struct sun4c_mmu_ring *rp = 0;
unsigned char ctx;
+#ifdef DEBUG_SUN4C_MM
+ int lim = num_contexts;
+#endif
/* If some are free, return first one. */
- if(sun4c_ufree_ring.num_entries)
+ if(sun4c_ufree_ring.num_entries) {
+#ifdef DEBUG_SUN4C_MM
+ if(sun4c_ufree_ring.ringhd.next == &sun4c_ufree_ring.ringhd)
+ panic("sun4c_user_strategy: num_entries!=0 but ring empty.");
+#endif
return sun4c_ufree_ring.ringhd.next;
+ }
if (sun4c_user_taken_entries) {
sun4c_shrink_kernel_ring();
+#ifdef DEBUG_SUN4C_MM
+ if(sun4c_ufree_ring.ringhd.next == &sun4c_ufree_ring.ringhd)
+ panic("sun4c_user_strategy: kernel shrunk but ufree empty.");
+#endif
return sun4c_ufree_ring.ringhd.next;
}
/* Grab one from the LRU context. */
next_one = ctx_used.next;
- while (sun4c_context_ring[next_one->ctx_number].num_entries == 0)
+ while ((sun4c_context_ring[next_one->ctx_number].num_entries == 0)
+#ifdef DEBUG_SUN4C_MM
+ && (--lim >= 0)
+#endif
+ )
next_one = next_one->next;
+#ifdef DEBUG_SUN4C_MM
+ if(lim < 0)
+ panic("No user segmaps!");
+#endif
+
ctx = next_one->ctx_number;
rp = &sun4c_context_ring[ctx];
sun4c_demap_one(rp, ctx);
+#ifdef DEBUG_SUN4C_MM
+ if(sun4c_ufree_ring.ringhd.next == &sun4c_ufree_ring.ringhd)
+ panic("sun4c_user_strategy: demapped one but ufree empty.");
+#endif
return sun4c_ufree_ring.ringhd.next;
}
@@ -839,13 +1038,34 @@
{
struct sun4c_mmu_entry *entry;
+#if 0
+ printk("grow: ");
+#endif
+
+ /* Prevent deadlock condition. */
+ if(sun4c_user_taken_entries >= max_user_taken_entries) {
+#if 0
+ printk("deadlock avoidance, taken= %d max= %d\n",
+ sun4c_user_taken_entries, max_user_taken_entries);
+#endif
+ return;
+ }
+
if (sun4c_ufree_ring.num_entries) {
entry = sun4c_ufree_ring.ringhd.next;
+#ifdef DEBUG_SUN4C_MM
+ if(entry == &sun4c_ufree_ring.ringhd)
+ panic("\nsun4c_grow_kernel_ring: num_entries!=0, ring empty.");
+#endif
remove_ring(&sun4c_ufree_ring, entry);
add_ring(&sun4c_kfree_ring, entry);
+#ifdef DEBUG_SUN4C_MM
+ if(sun4c_user_taken_entries < 0)
+ panic("\nsun4c_grow_kernel_ring: taken < 0.");
+#endif
sun4c_user_taken_entries++;
#if 0
- printk("grow: ufree= %d, kfree= %d, kernel= %d\n",
+ printk("ufree= %d, kfree= %d, kernel= %d\n",
sun4c_ufree_ring.num_entries,
sun4c_kfree_ring.num_entries,
sun4c_kernel_ring.num_entries);
@@ -856,12 +1076,14 @@
static inline void alloc_user_segment(unsigned long address, unsigned char ctx)
{
struct sun4c_mmu_entry *entry;
+ unsigned long flags;
- address &= SUN4C_REAL_PGDIR_MASK;
+ save_and_cli(flags);
entry = sun4c_user_strategy();
+ entry->vaddr = (address & SUN4C_REAL_PGDIR_MASK);
assign_user_entry(ctx, entry);
- entry->vaddr = address;
sun4c_user_map(entry);
+ restore_flags(flags);
}
/* This is now a fast in-window trap handler to avoid any and all races. */
@@ -896,6 +1118,8 @@
struct task_bucket *sun4c_bucket[NR_TASKS];
+static int sun4c_lowbucket_avail;
+
#define BUCKET_EMPTY ((struct task_bucket *) 0)
#define BUCKET_SIZE (PAGE_SIZE << 2)
#define BUCKET_SHIFT 14 /* log2(sizeof(struct task_bucket)) */
@@ -915,8 +1139,13 @@
addr &= SUN4C_REAL_PGDIR_MASK;
stolen = sun4c_user_strategy();
remove_ring(&sun4c_ufree_ring, stolen);
+ max_user_taken_entries--;
+#ifdef DEBUG_SUN4C_MM
+ if(max_user_taken_entries < 0)
+ panic("get_locked_segment: max_user_taken < 0.");
+#endif
stolen->vaddr = addr;
- flush_user_windows();
+ FUW_INLINE
sun4c_kernel_map(stolen);
restore_flags(flags);
}
@@ -931,60 +1160,52 @@
addr &= SUN4C_REAL_PGDIR_MASK;
pseg = sun4c_get_segmap(addr);
entry = &mmu_entry_pool[pseg];
- flush_user_windows();
+
+ FUW_INLINE
sun4c_flush_segment(addr);
sun4c_kernel_unmap(entry);
add_ring(&sun4c_ufree_ring, entry);
+#ifdef DEBUG_SUN4C_MM
+ if(max_user_taken_entries < 0)
+ panic("free_locked_segment: max_user_taken < 0.");
+#endif
+ max_user_taken_entries++;
restore_flags(flags);
}
static inline void garbage_collect(int entry)
{
- unsigned long flags;
int start, end;
- save_and_cli(flags);
-
/* 16 buckets per segment... */
entry &= ~15;
start = entry;
for(end = (start + 16); start < end; start++)
if(sun4c_bucket[start] != BUCKET_EMPTY)
- goto done;
+ return;
+
/* Entire segment empty, release it. */
free_locked_segment(BUCKET_ADDR(entry));
-done:
- restore_flags(flags);
}
static struct task_struct *sun4c_alloc_task_struct(void)
{
- unsigned long addr, page, flags;
+ unsigned long addr, page;
int entry;
- save_and_cli(flags);
-
page = get_free_page(GFP_KERNEL);
- if(!page) {
- restore_flags(flags);
+ if(!page)
return (struct task_struct *) 0;
- }
- /* XXX Bahh, linear search too slow, use hash
- * XXX table in final implementation. Or
- * XXX keep track of first free when we free
- * XXX a bucket... anything but this.
- */
- for(entry = 0; entry < NR_TASKS; entry++)
+
+ for(entry = sun4c_lowbucket_avail; entry < NR_TASKS; entry++)
if(sun4c_bucket[entry] == BUCKET_EMPTY)
break;
if(entry == NR_TASKS) {
free_page(page);
- restore_flags(flags);
return (struct task_struct *) 0;
}
-
- /* Prevent an alias from occurring while the page is ours. */
- sun4c_flush_page(page);
+ if(entry >= sun4c_lowbucket_avail)
+ sun4c_lowbucket_avail = entry + 1;
addr = BUCKET_ADDR(entry);
sun4c_bucket[entry] = (struct task_bucket *) addr;
@@ -992,8 +1213,6 @@
get_locked_segment(addr);
sun4c_put_pte(addr, BUCKET_PTE(page));
- restore_flags(flags);
-
return (struct task_struct *) addr;
}
@@ -1013,47 +1232,80 @@
return 0;
}
- /* Prevent aliases from occurring while the pages are ours. */
- sun4c_flush_page(page[0]);
- sun4c_flush_page(page[1]);
-
saddr += PAGE_SIZE << 1;
sun4c_put_pte(saddr, BUCKET_PTE(page[0]));
sun4c_put_pte(saddr + PAGE_SIZE, BUCKET_PTE(page[1]));
return saddr;
}
-static void sun4c_free_kernel_stack(unsigned long stack)
+static void sun4c_free_kernel_stack_hw(unsigned long stack)
{
unsigned long page[2];
- unsigned long flags;
- save_and_cli(flags);
page[0] = BUCKET_PTE_PAGE(sun4c_get_pte(stack));
page[1] = BUCKET_PTE_PAGE(sun4c_get_pte(stack+PAGE_SIZE));
- sun4c_flush_page(stack);
- sun4c_flush_page(stack + PAGE_SIZE);
+
+ /* We are deleting a mapping, so the flushes here are mandatory. */
+ sun4c_flush_page_hw(stack);
+ sun4c_flush_page_hw(stack + PAGE_SIZE);
+
sun4c_put_pte(stack, 0);
sun4c_put_pte(stack + PAGE_SIZE, 0);
free_page(page[0]);
free_page(page[1]);
- restore_flags(flags);
}
-static void sun4c_free_task_struct(struct task_struct *tsk)
+static void sun4c_free_task_struct_hw(struct task_struct *tsk)
{
unsigned long tsaddr = (unsigned long) tsk;
unsigned long page = BUCKET_PTE_PAGE(sun4c_get_pte(tsaddr));
- unsigned long flags;
int entry = BUCKET_NUM(tsaddr);
- save_and_cli(flags);
- sun4c_flush_page(tsaddr);
+ /* We are deleting a mapping, so the flush here is mandatory. */
+ sun4c_flush_page_hw(tsaddr);
+
sun4c_put_pte(tsaddr, 0);
sun4c_bucket[entry] = BUCKET_EMPTY;
+ if(entry < sun4c_lowbucket_avail)
+ sun4c_lowbucket_avail = entry;
+
+ free_page(page);
+ garbage_collect(entry);
+}
+
+static void sun4c_free_kernel_stack_sw(unsigned long stack)
+{
+ unsigned long page[2];
+
+ page[0] = BUCKET_PTE_PAGE(sun4c_get_pte(stack));
+ page[1] = BUCKET_PTE_PAGE(sun4c_get_pte(stack+PAGE_SIZE));
+
+ /* We are deleting a mapping, so the flushes here are mandatory. */
+ sun4c_flush_page_sw(stack);
+ sun4c_flush_page_sw(stack + PAGE_SIZE);
+
+ sun4c_put_pte(stack, 0);
+ sun4c_put_pte(stack + PAGE_SIZE, 0);
+ free_page(page[0]);
+ free_page(page[1]);
+}
+
+static void sun4c_free_task_struct_sw(struct task_struct *tsk)
+{
+ unsigned long tsaddr = (unsigned long) tsk;
+ unsigned long page = BUCKET_PTE_PAGE(sun4c_get_pte(tsaddr));
+ int entry = BUCKET_NUM(tsaddr);
+
+ /* We are deleting a mapping, so the flush here is mandatory. */
+ sun4c_flush_page_sw(tsaddr);
+
+ sun4c_put_pte(tsaddr, 0);
+ sun4c_bucket[entry] = BUCKET_EMPTY;
+ if(entry < sun4c_lowbucket_avail)
+ sun4c_lowbucket_avail = entry;
+
free_page(page);
garbage_collect(entry);
- restore_flags(flags);
}
__initfunc(static void sun4c_init_buckets(void))
@@ -1066,6 +1318,7 @@
}
for(entry = 0; entry < NR_TASKS; entry++)
sun4c_bucket[entry] = BUCKET_EMPTY;
+ sun4c_lowbucket_avail = 0;
}
static unsigned long sun4c_iobuffer_start;
@@ -1151,6 +1404,8 @@
save_and_cli(flags);
while (npages != 0) {
--npages;
+
+ /* This mapping is marked non-cachable, no flush necessary. */
sun4c_put_pte(vpage, 0);
clear_bit((vpage - sun4c_iobuffer_start) >> PAGE_SHIFT,
sun4c_iobuffer_map);
@@ -1176,39 +1431,37 @@
* by implication and fool the page locking code above
* if passed to by mistake.
*/
-static char *sun4c_get_scsi_one(char *bufptr, unsigned long len, struct linux_sbus *sbus)
+static __u32 sun4c_get_scsi_one(char *bufptr, unsigned long len, struct linux_sbus *sbus)
{
unsigned long page;
- page = ((unsigned long) bufptr) & PAGE_MASK;
+ page = ((unsigned long)bufptr) & PAGE_MASK;
if(MAP_NR(page) > max_mapnr) {
sun4c_flush_page(page);
- return bufptr; /* already locked */
+ return (__u32)bufptr; /* already locked */
}
- return sun4c_lockarea(bufptr, len);
+ return (__u32)sun4c_lockarea(bufptr, len);
}
static void sun4c_get_scsi_sgl(struct mmu_sglist *sg, int sz, struct linux_sbus *sbus)
{
while(sz >= 0) {
- sg[sz].dvma_addr = sun4c_lockarea(sg[sz].addr, sg[sz].len);
+ sg[sz].dvma_addr = (__u32)sun4c_lockarea(sg[sz].addr, sg[sz].len);
sz--;
}
}
-static void sun4c_release_scsi_one(char *bufptr, unsigned long len, struct linux_sbus *sbus)
+static void sun4c_release_scsi_one(__u32 bufptr, unsigned long len, struct linux_sbus *sbus)
{
- unsigned long page = (unsigned long) bufptr;
-
- if(page < sun4c_iobuffer_start)
+ if(bufptr < sun4c_iobuffer_start)
return; /* On kernel stack or similar, see above */
- sun4c_unlockarea(bufptr, len);
+ sun4c_unlockarea((char *)bufptr, len);
}
static void sun4c_release_scsi_sgl(struct mmu_sglist *sg, int sz, struct linux_sbus *sbus)
{
while(sz >= 0) {
- sun4c_unlockarea(sg[sz].dvma_addr, sg[sz].len);
+ sun4c_unlockarea((char *)sg[sz].dvma_addr, sg[sz].len);
sz--;
}
}
@@ -1256,14 +1509,12 @@
/* Cache flushing on the sun4c. */
static void sun4c_flush_cache_all(void)
{
- extern unsigned long bcopy;
- unsigned long begin, end, flags;
+ unsigned long begin, end;
- begin = ((unsigned long) &bcopy);
- end = (begin + sun4c_vacinfo.num_bytes);
+ FUW_INLINE
+ begin = (KERNBASE + SUN4C_REAL_PGDIR_SIZE);
+ end = (begin + SUN4C_VAC_SIZE);
- save_and_cli(flags);
- flush_user_windows();
if(sun4c_vacinfo.linesize == 32) {
while(begin < end) {
__asm__ __volatile__("
@@ -1309,106 +1560,236 @@
begin += 256;
}
}
- restore_flags(flags);
}
-static void sun4c_flush_cache_mm(struct mm_struct *mm)
+static void sun4c_flush_cache_mm_hw(struct mm_struct *mm)
{
- unsigned long flags;
- int octx, new_ctx = mm->context;
+ int new_ctx = mm->context;
+
+ if(new_ctx != NO_CONTEXT && sun4c_context_ring[new_ctx].num_entries) {
+ struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd;
+ unsigned long flags;
- if(new_ctx != NO_CONTEXT) {
save_and_cli(flags);
- octx = sun4c_get_context();
- flush_user_windows();
- sun4c_set_context(new_ctx);
- sun4c_flush_context();
- sun4c_set_context(octx);
+ if(head->next != head) {
+ struct sun4c_mmu_entry *entry = head->next;
+ int savectx = sun4c_get_context();
+
+ FUW_INLINE
+ sun4c_set_context(new_ctx);
+ sun4c_flush_context_hw();
+ do {
+ struct sun4c_mmu_entry *next = entry->next;
+
+ sun4c_user_unmap(entry);
+ free_user_entry(new_ctx, entry);
+
+ entry = next;
+ } while(entry != head);
+ sun4c_set_context(savectx);
+ }
restore_flags(flags);
}
}
-static void sun4c_flush_cache_range(struct mm_struct *mm, unsigned long start, unsigned long end)
+static void sun4c_flush_cache_range_hw(struct mm_struct *mm, unsigned long start, unsigned long end)
{
- int size, size2, octx, i, new_ctx = mm->context;
- unsigned long start2, end2, flags;
- struct sun4c_mmu_entry *entry, *entry2;
+ int new_ctx = mm->context;
#if KGPROF_PROFILING
kgprof_profile();
#endif
-
if(new_ctx != NO_CONTEXT) {
+ struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd;
+ struct sun4c_mmu_entry *entry;
+ unsigned long flags;
+
+ FUW_INLINE
save_and_cli(flags);
- size = end - start;
+ /* All user segmap chains are ordered on entry->vaddr. */
+ for(entry = head->next;
+ (entry != head) && ((entry->vaddr+SUN4C_REAL_PGDIR_SIZE) < start);
+ entry = entry->next)
+ ;
+
+ /* Tracing various job mixtures showed that this conditional
+ * only passes ~35% of the time for most worse case situations,
+ * therefore we avoid all of this gross overhead ~65% of the time.
+ */
+ if((entry != head) && (entry->vaddr < end)) {
+ int octx = sun4c_get_context();
+ sun4c_set_context(new_ctx);
+
+ /* At this point, always, (start >= entry->vaddr) and
+ * (entry->vaddr < end), once the latter condition
+ * ceases to hold, or we hit the end of the list, we
+ * exit the loop. The ordering of all user allocated
+ * segmaps makes this all work out so beautifully.
+ */
+ do {
+ struct sun4c_mmu_entry *next = entry->next;
+ unsigned long realend;
+
+ /* "realstart" is always >= entry->vaddr */
+ realend = entry->vaddr + SUN4C_REAL_PGDIR_SIZE;
+ if(end < realend)
+ realend = end;
+ if((realend - entry->vaddr) <= (PAGE_SIZE << 3)) {
+ unsigned long page = entry->vaddr;
+ while(page < realend) {
+ sun4c_flush_page_hw(page);
+ page += PAGE_SIZE;
+ }
+ } else {
+ sun4c_flush_segment_hw(entry->vaddr);
+ sun4c_user_unmap(entry);
+ free_user_entry(new_ctx, entry);
+ }
+ entry = next;
+ } while((entry != head) && (entry->vaddr < end));
+ sun4c_set_context(octx);
+ }
+ restore_flags(flags);
+ }
+}
+
+/* XXX no save_and_cli/restore_flags needed, but put here if darkside still crashes */
+static void sun4c_flush_cache_page_hw(struct vm_area_struct *vma, unsigned long page)
+{
+ struct mm_struct *mm = vma->vm_mm;
+ int new_ctx = mm->context;
- octx = sun4c_get_context();
- flush_user_windows();
+ /* Sun4c has no separate I/D caches so cannot optimize for non
+ * text page flushes.
+ */
+ if(new_ctx != NO_CONTEXT) {
+ int octx = sun4c_get_context();
+
+ FUW_INLINE
sun4c_set_context(new_ctx);
+ sun4c_flush_page_hw(page);
+ sun4c_set_context(octx);
+ }
+}
+
+static void sun4c_flush_page_to_ram_hw(unsigned long page)
+{
+ sun4c_flush_page_hw(page);
+}
+
+static void sun4c_flush_cache_mm_sw(struct mm_struct *mm)
+{
+ int new_ctx = mm->context;
+
+ if(new_ctx != NO_CONTEXT && sun4c_context_ring[new_ctx].num_entries) {
+ struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd;
+ unsigned long flags;
+
+ save_and_cli(flags);
+ if(head->next != head) {
+ struct sun4c_mmu_entry *entry = head->next;
+ int savectx = sun4c_get_context();
+
+ FUW_INLINE
+ sun4c_set_context(new_ctx);
+ sun4c_flush_context_sw();
+ do {
+ struct sun4c_mmu_entry *next = entry->next;
- entry = sun4c_context_ring[new_ctx].ringhd.next;
- i = sun4c_context_ring[new_ctx].num_entries;
- while (i--) {
- entry2 = entry->next;
- if (entry->vaddr < start || entry->vaddr >= end)
- goto next_entry;
-
- start2 = MAX(start,entry->vaddr);
- end2 = MIN(end,entry->vaddr+SUN4C_REAL_PGDIR_SIZE);
- size2 = end2 - start2;
-
- if (size2 <= (PAGE_SIZE << 3)) {
- start2 &= PAGE_MASK;
- while(start2 < end2) {
- sun4c_flush_page(start2);
- start2 += PAGE_SIZE;
- }
- } else {
- start2 &= SUN4C_REAL_PGDIR_MASK;
- sun4c_flush_segment(start2);
-
- /* We are betting that the entry will not be
- * needed for a while.
- */
sun4c_user_unmap(entry);
free_user_entry(new_ctx, entry);
- }
- next_entry:
- entry = entry2;
+ entry = next;
+ } while(entry != head);
+ sun4c_set_context(savectx);
}
- sun4c_set_context(octx);
restore_flags(flags);
}
}
-static void sun4c_flush_cache_page(struct vm_area_struct *vma, unsigned long page)
+static void sun4c_flush_cache_range_sw(struct mm_struct *mm, unsigned long start, unsigned long end)
+{
+ int new_ctx = mm->context;
+
+#if KGPROF_PROFILING
+ kgprof_profile();
+#endif
+ if(new_ctx != NO_CONTEXT) {
+ struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd;
+ struct sun4c_mmu_entry *entry;
+ unsigned long flags;
+
+ FUW_INLINE
+ save_and_cli(flags);
+ /* All user segmap chains are ordered on entry->vaddr. */
+ for(entry = head->next;
+ (entry != head) && ((entry->vaddr+SUN4C_REAL_PGDIR_SIZE) < start);
+ entry = entry->next)
+ ;
+
+ /* Tracing various job mixtures showed that this conditional
+ * only passes ~35% of the time for most worse case situations,
+ * therefore we avoid all of this gross overhead ~65% of the time.
+ */
+ if((entry != head) && (entry->vaddr < end)) {
+ int octx = sun4c_get_context();
+ sun4c_set_context(new_ctx);
+
+ /* At this point, always, (start >= entry->vaddr) and
+ * (entry->vaddr < end), once the latter condition
+ * ceases to hold, or we hit the end of the list, we
+ * exit the loop. The ordering of all user allocated
+ * segmaps makes this all work out so beautifully.
+ */
+ do {
+ struct sun4c_mmu_entry *next = entry->next;
+ unsigned long realend;
+
+ /* "realstart" is always >= entry->vaddr */
+ realend = entry->vaddr + SUN4C_REAL_PGDIR_SIZE;
+ if(end < realend)
+ realend = end;
+ if((realend - entry->vaddr) <= (PAGE_SIZE << 3)) {
+ unsigned long page = entry->vaddr;
+ while(page < realend) {
+ sun4c_flush_page_sw(page);
+ page += PAGE_SIZE;
+ }
+ } else {
+ sun4c_flush_segment_sw(entry->vaddr);
+ sun4c_user_unmap(entry);
+ free_user_entry(new_ctx, entry);
+ }
+ entry = next;
+ } while((entry != head) && (entry->vaddr < end));
+ sun4c_set_context(octx);
+ }
+ restore_flags(flags);
+ }
+}
+
+static void sun4c_flush_cache_page_sw(struct vm_area_struct *vma, unsigned long page)
{
- unsigned long flags;
struct mm_struct *mm = vma->vm_mm;
- int octx, new_ctx = mm->context;
+ int new_ctx = mm->context;
/* Sun4c has no separate I/D caches so cannot optimize for non
* text page flushes.
*/
if(new_ctx != NO_CONTEXT) {
- save_and_cli(flags);
- octx = sun4c_get_context();
- flush_user_windows();
+ int octx = sun4c_get_context();
+
+ FUW_INLINE
sun4c_set_context(new_ctx);
- sun4c_flush_page(page);
+ sun4c_flush_page_sw(page);
sun4c_set_context(octx);
- restore_flags(flags);
}
}
-/* Even though sun4c is write through, a virtual cache alias inconsistancy
- * can still occur with COW page handling so we must flush anyways.
- */
-static void sun4c_flush_page_to_ram(unsigned long page)
+static void sun4c_flush_page_to_ram_sw(unsigned long page)
{
- sun4c_flush_page(page);
+ sun4c_flush_page_sw(page);
}
/* Sun4c cache is unified, both instructions and data live there, so
@@ -1447,82 +1828,178 @@
restore_flags(flags);
}
-static void sun4c_flush_tlb_mm(struct mm_struct *mm)
+static void sun4c_flush_tlb_mm_hw(struct mm_struct *mm)
{
- struct sun4c_mmu_entry *this_entry, *next_entry;
- struct sun4c_mmu_ring *crp;
- unsigned long flags;
- int savectx, new_ctx = mm->context;
+ int new_ctx = mm->context;
if(new_ctx != NO_CONTEXT) {
+ struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd;
+ unsigned long flags;
+
save_and_cli(flags);
- crp = &sun4c_context_ring[new_ctx];
- savectx = sun4c_get_context();
- this_entry = crp->ringhd.next;
- flush_user_windows();
- sun4c_set_context(new_ctx);
- while(crp->num_entries) {
- next_entry = this_entry->next;
- sun4c_flush_segment(this_entry->vaddr);
- sun4c_user_unmap(this_entry);
- free_user_entry(new_ctx, this_entry);
- this_entry = next_entry;
+ if(head->next != head) {
+ struct sun4c_mmu_entry *entry = head->next;
+ int savectx = sun4c_get_context();
+
+ FUW_INLINE
+ sun4c_set_context(new_ctx);
+ sun4c_flush_context_hw();
+ do {
+ struct sun4c_mmu_entry *next = entry->next;
+
+ sun4c_user_unmap(entry);
+ free_user_entry(new_ctx, entry);
+
+ entry = next;
+ } while(entry != head);
+ sun4c_set_context(savectx);
+ }
+ restore_flags(flags);
+ }
+}
+
+static void sun4c_flush_tlb_range_hw(struct mm_struct *mm, unsigned long start, unsigned long end)
+{
+ int new_ctx = mm->context;
+
+ if(new_ctx != NO_CONTEXT) {
+ struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd;
+ struct sun4c_mmu_entry *entry;
+ unsigned long flags;
+#if KGPROF_PROFILING
+ kgprof_profile();
+#endif
+
+ save_and_cli(flags);
+ /* See commentary in sun4c_flush_cache_range_*(). */
+ for(entry = head->next;
+ (entry != head) && ((entry->vaddr+SUN4C_REAL_PGDIR_SIZE) < start);
+ entry = entry->next)
+ ;
+
+ if((entry != head) && (entry->vaddr < end)) {
+ int octx = sun4c_get_context();
+
+ /* This window flush is paranoid I think... -DaveM */
+ FUW_INLINE
+ sun4c_set_context(new_ctx);
+ do {
+ struct sun4c_mmu_entry *next = entry->next;
+
+ sun4c_flush_segment_hw(entry->vaddr);
+ sun4c_user_unmap(entry);
+ free_user_entry(new_ctx, entry);
+
+ entry = next;
+ } while((entry != head) && (entry->vaddr < end));
+ sun4c_set_context(octx);
}
+ restore_flags(flags);
+ }
+}
+
+static void sun4c_flush_tlb_page_hw(struct vm_area_struct *vma, unsigned long page)
+{
+ struct mm_struct *mm = vma->vm_mm;
+ int new_ctx = mm->context;
+
+ if(new_ctx != NO_CONTEXT) {
+ int savectx = sun4c_get_context();
+
+ FUW_INLINE
+ sun4c_set_context(new_ctx);
+ page &= PAGE_MASK;
+ sun4c_flush_page_hw(page);
+ sun4c_put_pte(page, 0);
sun4c_set_context(savectx);
+ }
+}
+
+static void sun4c_flush_tlb_mm_sw(struct mm_struct *mm)
+{
+ int new_ctx = mm->context;
+
+ if(new_ctx != NO_CONTEXT) {
+ struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd;
+ unsigned long flags;
+
+ save_and_cli(flags);
+ if(head->next != head) {
+ struct sun4c_mmu_entry *entry = head->next;
+ int savectx = sun4c_get_context();
+
+ FUW_INLINE
+ sun4c_set_context(new_ctx);
+ sun4c_flush_context_sw();
+ do {
+ struct sun4c_mmu_entry *next = entry->next;
+
+ sun4c_user_unmap(entry);
+ free_user_entry(new_ctx, entry);
+
+ entry = next;
+ } while(entry != head);
+ sun4c_set_context(savectx);
+ }
restore_flags(flags);
}
}
-static void sun4c_flush_tlb_range(struct mm_struct *mm, unsigned long start, unsigned long end)
+static void sun4c_flush_tlb_range_sw(struct mm_struct *mm, unsigned long start, unsigned long end)
{
- struct sun4c_mmu_entry *entry, *entry2;
- unsigned long flags;
- int i, savectx, new_ctx = mm->context;
+ int new_ctx = mm->context;
- if(new_ctx == NO_CONTEXT)
- return;
+ if(new_ctx != NO_CONTEXT) {
+ struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd;
+ struct sun4c_mmu_entry *entry;
+ unsigned long flags;
#if KGPROF_PROFILING
- kgprof_profile();
+ kgprof_profile();
#endif
- save_and_cli(flags);
- savectx = sun4c_get_context();
- flush_user_windows();
- sun4c_set_context(new_ctx);
- start &= SUN4C_REAL_PGDIR_MASK;
+ save_and_cli(flags);
+ /* See commentary in sun4c_flush_cache_range_*(). */
+ for(entry = head->next;
+ (entry != head) && ((entry->vaddr+SUN4C_REAL_PGDIR_SIZE) < start);
+ entry = entry->next)
+ ;
+
+ if((entry != head) && (entry->vaddr < end)) {
+ int octx = sun4c_get_context();
+
+ /* This window flush is paranoid I think... -DaveM */
+ FUW_INLINE
+ sun4c_set_context(new_ctx);
+ do {
+ struct sun4c_mmu_entry *next = entry->next;
- entry = sun4c_context_ring[new_ctx].ringhd.next;
- i = sun4c_context_ring[new_ctx].num_entries;
- while (i--) {
- entry2 = entry->next;
- if (entry->vaddr >= start && entry->vaddr < end) {
- sun4c_flush_segment(entry->vaddr);
- sun4c_user_unmap(entry);
- free_user_entry(new_ctx, entry);
+ sun4c_flush_segment_sw(entry->vaddr);
+ sun4c_user_unmap(entry);
+ free_user_entry(new_ctx, entry);
+
+ entry = next;
+ } while((entry != head) && (entry->vaddr < end));
+ sun4c_set_context(octx);
}
- entry = entry2;
+ restore_flags(flags);
}
- sun4c_set_context(savectx);
- restore_flags(flags);
}
-static void sun4c_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
+static void sun4c_flush_tlb_page_sw(struct vm_area_struct *vma, unsigned long page)
{
struct mm_struct *mm = vma->vm_mm;
- unsigned long flags;
- int savectx, new_ctx = mm->context;
+ int new_ctx = mm->context;
if(new_ctx != NO_CONTEXT) {
- save_and_cli(flags);
- savectx = sun4c_get_context();
- flush_user_windows();
+ int savectx = sun4c_get_context();
+
+ FUW_INLINE
sun4c_set_context(new_ctx);
page &= PAGE_MASK;
- sun4c_flush_page(page);
+ sun4c_flush_page_sw(page);
sun4c_put_pte(page, 0);
sun4c_set_context(savectx);
- restore_flags(flags);
}
}
@@ -1548,7 +2025,7 @@
sun4c_put_pte(virt_addr, 0);
}
-static inline void sun4c_alloc_context(struct mm_struct *mm)
+static void sun4c_alloc_context_hw(struct mm_struct *mm)
{
struct ctx_list *ctxp;
@@ -1563,74 +2040,114 @@
ctxp = ctx_used.next;
if(ctxp->ctx_mm == current->mm)
ctxp = ctxp->next;
+#ifdef DEBUG_SUN4C_MM
if(ctxp == &ctx_used)
panic("out of mmu contexts");
+#endif
remove_from_ctx_list(ctxp);
add_to_used_ctxlist(ctxp);
ctxp->ctx_mm->context = NO_CONTEXT;
ctxp->ctx_mm = mm;
mm->context = ctxp->ctx_number;
- sun4c_demap_context(&sun4c_context_ring[ctxp->ctx_number],
+ sun4c_demap_context_hw(&sun4c_context_ring[ctxp->ctx_number],
ctxp->ctx_number);
}
-#if some_day_soon /* We need some tweaking to start using this */
-extern void force_user_fault(unsigned long, int);
+static void sun4c_switch_to_context_hw(struct task_struct *tsk)
+{
+ struct ctx_list *ctx;
+
+ if(tsk->mm->context == NO_CONTEXT) {
+ sun4c_alloc_context_hw(tsk->mm);
+ } else {
+ /* Update the LRU ring of contexts. */
+ ctx = ctx_list_pool + tsk->mm->context;
+ remove_from_ctx_list(ctx);
+ add_to_used_ctxlist(ctx);
+ }
+ sun4c_set_context(tsk->mm->context);
+}
+
+static void sun4c_init_new_context_hw(struct mm_struct *mm)
+{
+ sun4c_alloc_context_hw(mm);
+ if(mm == current->mm)
+ sun4c_set_context(mm->context);
+}
-void sun4c_switch_heuristic(struct pt_regs *regs)
+static void sun4c_destroy_context_hw(struct mm_struct *mm)
{
- unsigned long sp = regs->u_regs[UREG_FP];
- unsigned long sp2 = sp + REGWIN_SZ - 0x8;
+ struct ctx_list *ctx_old;
- force_user_fault(regs->pc, 0);
- force_user_fault(sp, 0);
- if((sp&PAGE_MASK) != (sp2&PAGE_MASK))
- force_user_fault(sp2, 0);
+ if(mm->context != NO_CONTEXT && mm->count == 1) {
+ sun4c_demap_context_hw(&sun4c_context_ring[mm->context], mm->context);
+ ctx_old = ctx_list_pool + mm->context;
+ remove_from_ctx_list(ctx_old);
+ add_to_free_ctxlist(ctx_old);
+ mm->context = NO_CONTEXT;
+ }
}
+
+static void sun4c_alloc_context_sw(struct mm_struct *mm)
+{
+ struct ctx_list *ctxp;
+
+ ctxp = ctx_free.next;
+ if(ctxp != &ctx_free) {
+ remove_from_ctx_list(ctxp);
+ add_to_used_ctxlist(ctxp);
+ mm->context = ctxp->ctx_number;
+ ctxp->ctx_mm = mm;
+ return;
+ }
+ ctxp = ctx_used.next;
+ if(ctxp->ctx_mm == current->mm)
+ ctxp = ctxp->next;
+#ifdef DEBUG_SUN4C_MM
+ if(ctxp == &ctx_used)
+ panic("out of mmu contexts");
#endif
+ remove_from_ctx_list(ctxp);
+ add_to_used_ctxlist(ctxp);
+ ctxp->ctx_mm->context = NO_CONTEXT;
+ ctxp->ctx_mm = mm;
+ mm->context = ctxp->ctx_number;
+ sun4c_demap_context_sw(&sun4c_context_ring[ctxp->ctx_number],
+ ctxp->ctx_number);
+}
-static void sun4c_switch_to_context(struct task_struct *tsk)
+static void sun4c_switch_to_context_sw(struct task_struct *tsk)
{
struct ctx_list *ctx;
- unsigned long flags;
- save_and_cli(flags);
if(tsk->mm->context == NO_CONTEXT) {
- sun4c_alloc_context(tsk->mm);
- goto set_context;
+ sun4c_alloc_context_sw(tsk->mm);
+ } else {
+ /* Update the LRU ring of contexts. */
+ ctx = ctx_list_pool + tsk->mm->context;
+ remove_from_ctx_list(ctx);
+ add_to_used_ctxlist(ctx);
}
-
- /* Update the LRU ring of contexts. */
- ctx = ctx_list_pool + tsk->mm->context;
- remove_from_ctx_list(ctx);
- add_to_used_ctxlist(ctx);
-
-set_context:
sun4c_set_context(tsk->mm->context);
- restore_flags(flags);
}
-static void sun4c_init_new_context(struct mm_struct *mm)
+static void sun4c_init_new_context_sw(struct mm_struct *mm)
{
- sun4c_alloc_context(mm);
+ sun4c_alloc_context_sw(mm);
if(mm == current->mm)
sun4c_set_context(mm->context);
}
-static void sun4c_destroy_context(struct mm_struct *mm)
+static void sun4c_destroy_context_sw(struct mm_struct *mm)
{
struct ctx_list *ctx_old;
if(mm->context != NO_CONTEXT && mm->count == 1) {
- unsigned long flags;
-
- save_and_cli(flags);
- sun4c_demap_context(&sun4c_context_ring[mm->context], mm->context);
+ sun4c_demap_context_sw(&sun4c_context_ring[mm->context], mm->context);
ctx_old = ctx_list_pool + mm->context;
remove_from_ctx_list(ctx_old);
add_to_free_ctxlist(ctx_old);
mm->context = NO_CONTEXT;
- restore_flags(flags);
}
}
@@ -1657,6 +2174,8 @@
"kfreepsegs\t: %d\n"
"usedpsegs\t: %d\n"
"ufreepsegs\t: %d\n"
+ "user_taken\t: %d\n"
+ "max_taken\t: %d\n"
"context\t\t: %d flushes\n"
"segment\t\t: %d flushes\n"
"page\t\t: %d flushes\n",
@@ -1669,6 +2188,8 @@
sun4c_kfree_ring.num_entries,
used_user_entries,
sun4c_ufree_ring.num_entries,
+ sun4c_user_taken_entries,
+ max_user_taken_entries,
ctxflushes, segflushes, pageflushes);
#if KGPROF_PROFILING
@@ -1971,31 +2492,20 @@
* now, so our ref/mod bit tracking quick userfaults eat a few more
* cycles than they used to.
*/
-void sun4c_update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t pte)
+static void sun4c_vac_alias_fixup(struct vm_area_struct *vma, unsigned long address, pte_t pte)
{
- unsigned long offset, vaddr, start, flags;
+ struct inode *inode;
pgd_t *pgdp;
pte_t *ptep;
- save_and_cli(flags);
- address &= PAGE_MASK;
- if(sun4c_get_segmap(address) == invalid_segment)
- alloc_user_segment(address, sun4c_get_context());
-
- if((vma->vm_flags & (VM_WRITE|VM_SHARED)) == (VM_WRITE|VM_SHARED)) {
- struct vm_area_struct *vmaring;
- struct inode *inode;
+ inode = vma->vm_inode;
+ if(inode) {
+ unsigned long offset = (address & PAGE_MASK) - vma->vm_start;
+ struct vm_area_struct *vmaring = inode->i_mmap;
int alias_found = 0;
-
- /* While on the other hand, this is very uncommon... */
- inode = vma->vm_inode;
- if(!inode)
- goto done;
-
- offset = (address & PAGE_MASK) - vma->vm_start;
- vmaring = inode->i_mmap;
do {
- vaddr = vmaring->vm_start + offset;
+ unsigned long vaddr = vmaring->vm_start + offset;
+ unsigned long start;
if (S4CVAC_BADALIAS(vaddr, address)) {
alias_found++;
@@ -2007,14 +2517,10 @@
if(!ptep) goto next;
if(pte_val(*ptep) & _SUN4C_PAGE_PRESENT) {
-#if 1
- printk("Fixing USER/USER alias [%ld:%08lx]\n",
- vmaring->vm_mm->context, start);
-#endif
- sun4c_flush_cache_page(vmaring, start);
- *ptep = __pte(pte_val(*ptep) |
- _SUN4C_PAGE_NOCACHE);
- sun4c_flush_tlb_page(vmaring, start);
+ flush_cache_page(vmaring, start);
+ pte_val(*ptep) = (pte_val(*ptep) |
+ _SUN4C_PAGE_NOCACHE);
+ flush_tlb_page(vmaring, start);
}
next:
start += PAGE_SIZE;
@@ -2025,11 +2531,24 @@
if(alias_found && !(pte_val(pte) & _SUN4C_PAGE_NOCACHE)) {
pgdp = sun4c_pgd_offset(vma->vm_mm, address);
ptep = sun4c_pte_offset((pmd_t *) pgdp, address);
- *ptep = __pte(pte_val(*ptep) | _SUN4C_PAGE_NOCACHE);
+ pte_val(*ptep) = (pte_val(*ptep) | _SUN4C_PAGE_NOCACHE);
pte = pte_val(*ptep);
}
}
-done:
+}
+
+void sun4c_update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t pte)
+{
+ unsigned long flags;
+
+ save_and_cli(flags);
+ address &= PAGE_MASK;
+ if(sun4c_get_segmap(address) == invalid_segment)
+ alloc_user_segment(address, sun4c_get_context());
+
+ if((vma->vm_flags & (VM_WRITE|VM_SHARED)) == (VM_WRITE|VM_SHARED))
+ sun4c_vac_alias_fixup(vma, address, pte);
+
sun4c_put_pte(address, pte_val(pte));
restore_flags(flags);
}
@@ -2045,7 +2564,7 @@
extern unsigned long sparc_iobase_vaddr;
kernel_end = (unsigned long) &end;
- kernel_end += (SUN4C_REAL_PGDIR_SIZE * 3);
+ kernel_end += (SUN4C_REAL_PGDIR_SIZE * 4);
kernel_end = SUN4C_REAL_PGDIR_ALIGN(kernel_end);
sun4c_probe_mmu();
invalid_segment = (num_segmaps - 1);
@@ -2083,6 +2602,9 @@
for(i = 0; i < num_segmaps; i++)
if(mmu_entry_pool[i].locked)
cnt++;
+
+ max_user_taken_entries = num_segmaps - cnt - 40 - 1;
+
printk("SUN4C: %d mmu entries for the kernel\n", cnt);
return start_mem;
}
@@ -2114,21 +2636,40 @@
/* Functions */
flush_cache_all = sun4c_flush_cache_all;
- flush_cache_mm = sun4c_flush_cache_mm;
- flush_cache_range = sun4c_flush_cache_range;
- flush_cache_page = sun4c_flush_cache_page;
- flush_tlb_all = sun4c_flush_tlb_all;
- flush_tlb_mm = sun4c_flush_tlb_mm;
- flush_tlb_range = sun4c_flush_tlb_range;
- flush_tlb_page = sun4c_flush_tlb_page;
+ if(sun4c_vacinfo.do_hwflushes) {
+ flush_cache_mm = sun4c_flush_cache_mm_hw;
+ flush_cache_range = sun4c_flush_cache_range_hw;
+ flush_cache_page = sun4c_flush_cache_page_hw;
+ flush_page_to_ram = sun4c_flush_page_to_ram_hw;
+ flush_tlb_mm = sun4c_flush_tlb_mm_hw;
+ flush_tlb_range = sun4c_flush_tlb_range_hw;
+ flush_tlb_page = sun4c_flush_tlb_page_hw;
+ free_kernel_stack = sun4c_free_kernel_stack_hw;
+ free_task_struct = sun4c_free_task_struct_hw;
+ switch_to_context = sun4c_switch_to_context_hw;
+ destroy_context = sun4c_destroy_context_hw;
+ init_new_context = sun4c_init_new_context_hw;
+ } else {
+ flush_cache_mm = sun4c_flush_cache_mm_sw;
+ flush_cache_range = sun4c_flush_cache_range_sw;
+ flush_cache_page = sun4c_flush_cache_page_sw;
+ flush_page_to_ram = sun4c_flush_page_to_ram_sw;
+ flush_tlb_mm = sun4c_flush_tlb_mm_sw;
+ flush_tlb_range = sun4c_flush_tlb_range_sw;
+ flush_tlb_page = sun4c_flush_tlb_page_sw;
+ free_kernel_stack = sun4c_free_kernel_stack_sw;
+ free_task_struct = sun4c_free_task_struct_sw;
+ switch_to_context = sun4c_switch_to_context_sw;
+ destroy_context = sun4c_destroy_context_sw;
+ init_new_context = sun4c_init_new_context_sw;
+ }
- flush_page_to_ram = sun4c_flush_page_to_ram;
+ flush_tlb_all = sun4c_flush_tlb_all;
flush_sig_insns = sun4c_flush_sig_insns;
set_pte = sun4c_set_pte;
- switch_to_context = sun4c_switch_to_context;
pmd_align = sun4c_pmd_align;
pgdir_align = sun4c_pgdir_align;
vmalloc_start = sun4c_vmalloc_start;
@@ -2180,8 +2721,7 @@
pte_mkdirty = sun4c_pte_mkdirty;
pte_mkyoung = sun4c_pte_mkyoung;
update_mmu_cache = sun4c_update_mmu_cache;
- destroy_context = sun4c_destroy_context;
- init_new_context = sun4c_init_new_context;
+
mmu_lockarea = sun4c_lockarea;
mmu_unlockarea = sun4c_unlockarea;
@@ -2198,8 +2738,6 @@
/* Task struct and kernel stack allocating/freeing. */
alloc_kernel_stack = sun4c_alloc_kernel_stack;
alloc_task_struct = sun4c_alloc_task_struct;
- free_kernel_stack = sun4c_free_kernel_stack;
- free_task_struct = sun4c_free_task_struct;
quick_kernel_fault = sun4c_quick_kernel_fault;
mmu_info = sun4c_mmu_info;
FUNET's LINUX-ADM group, linux-adm@nic.funet.fi
TCL-scripts by Sam Shen, slshen@lbl.gov