patch-2.1.16 linux/arch/sparc/mm/sun4c.c
Next file: linux/arch/sparc/prom/init.c
Previous file: linux/arch/sparc/mm/srmmu.c
Back to the patch index
Back to the overall index
- Lines: 1673
- Date:
Fri Dec 13 11:37:32 1996
- Orig file:
v2.1.15/linux/arch/sparc/mm/sun4c.c
- Orig date:
Tue Nov 12 15:56:04 1996
diff -u --recursive --new-file v2.1.15/linux/arch/sparc/mm/sun4c.c linux/arch/sparc/mm/sun4c.c
@@ -1,4 +1,4 @@
-/* $Id: sun4c.c,v 1.121 1996/11/01 20:36:27 ecd Exp $
+/* $Id: sun4c.c,v 1.132 1996/12/10 06:06:27 davem Exp $
* sun4c.c: Doing in software what should be done in hardware.
*
* Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu)
@@ -8,6 +8,7 @@
#include <linux/kernel.h>
#include <linux/mm.h>
+#include <linux/init.h>
#include <asm/page.h>
#include <asm/pgtable.h>
@@ -23,19 +24,7 @@
extern int num_segmaps, num_contexts;
-/* Small structure for ease of handling in the low level kernel fault
- * handler. This holds all information necessary, like the sun4c_ufree_ring
- * for user segments.
- */
-struct sun4c_segment_info {
- unsigned long vaddr;
- unsigned char pseg;
-};
-struct sun4c_segment_info *sun4c_kernel_next;
-
#define SUN4C_KERNEL_BUCKETS 32
-#define SUN4C_KERNEL_BSIZE (sizeof(struct sun4c_segment_info) \
- * SUN4C_KERNEL_BUCKETS)
#ifndef MAX
#define MAX(a,b) ((a)<(b)?(b):(a))
@@ -45,7 +34,6 @@
#endif
-
#define KGPROF_PROFILING 0
#if KGPROF_PROFILING
#define KGPROF_DEPTH 3 /* this needs to match the code below */
@@ -85,6 +73,7 @@
/* Flushing the cache. */
struct sun4c_vac_props sun4c_vacinfo;
static int ctxflushes, segflushes, pageflushes;
+unsigned long sun4c_kernel_faults;
/* convert a virtual address to a physical address and vice
versa. Easy on the 4c */
@@ -121,39 +110,61 @@
/* Blow the entire current context out of the virtual cache. */
static inline void sun4c_flush_context(void)
{
- unsigned long vaddr;
+ extern unsigned long bcopy;
+ unsigned long begin, end, flags;
ctxflushes++;
- if(sun4c_vacinfo.do_hwflushes) {
- for(vaddr=0; vaddr < sun4c_vacinfo.num_bytes; vaddr+=PAGE_SIZE)
- __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : :
- "r" (vaddr), "i" (ASI_HWFLUSHCONTEXT));
+
+ begin = ((unsigned long) &bcopy);
+ end = (begin + sun4c_vacinfo.num_bytes);
+
+ save_and_cli(flags);
+ if(sun4c_vacinfo.linesize == 32) {
+ while(begin < end) {
+ __asm__ __volatile__("
+ ld [%0 + 0x00], %%g0
+ ld [%0 + 0x20], %%g0
+ ld [%0 + 0x40], %%g0
+ ld [%0 + 0x60], %%g0
+ ld [%0 + 0x80], %%g0
+ ld [%0 + 0xa0], %%g0
+ ld [%0 + 0xc0], %%g0
+ ld [%0 + 0xe0], %%g0
+ ld [%0 + 0x100], %%g0
+ ld [%0 + 0x120], %%g0
+ ld [%0 + 0x140], %%g0
+ ld [%0 + 0x160], %%g0
+ ld [%0 + 0x180], %%g0
+ ld [%0 + 0x1a0], %%g0
+ ld [%0 + 0x1c0], %%g0
+ ld [%0 + 0x1e0], %%g0
+ " : : "r" (begin));
+ begin += 512;
+ }
} else {
- /* AJT: possibly read the tags and avoid flushing the ones that
- are above 0xf0000000 so the kernel isn't flushed all the time */
- __asm__ __volatile__("add %1, %1, %%g1\n\t"
- "add %1, %%g1, %%g2\n\t"
- "add %1, %%g2, %%g3\n\t"
- "add %1, %%g3, %%g4\n\t"
- "add %1, %%g4, %%g5\n\t"
- "add %1, %%g5, %%o4\n\t"
- "add %1, %%o4, %%o5\n"
- "1:\n\t"
- "subcc %0, %%o5, %0\n\t"
- "sta %%g0, [%0] %2\n\t"
- "sta %%g0, [%0 + %1] %2\n\t"
- "sta %%g0, [%0 + %%g1] %2\n\t"
- "sta %%g0, [%0 + %%g2] %2\n\t"
- "sta %%g0, [%0 + %%g3] %2\n\t"
- "sta %%g0, [%0 + %%g4] %2\n\t"
- "sta %%g0, [%0 + %%g5] %2\n\t"
- "bg 1b\n\t"
- " sta %%g0, [%0 + %%o4] %2\n\t" : :
- "r" (sun4c_vacinfo.num_bytes),
- "r" (sun4c_vacinfo.linesize),
- "i" (ASI_FLUSHCTX) :
- "g1", "g2", "g3", "g4", "g5", "o4", "o5");
+ while(begin < end) {
+ __asm__ __volatile__("
+ ld [%0 + 0x00], %%g0
+ ld [%0 + 0x10], %%g0
+ ld [%0 + 0x20], %%g0
+ ld [%0 + 0x30], %%g0
+ ld [%0 + 0x40], %%g0
+ ld [%0 + 0x50], %%g0
+ ld [%0 + 0x60], %%g0
+ ld [%0 + 0x70], %%g0
+ ld [%0 + 0x80], %%g0
+ ld [%0 + 0x90], %%g0
+ ld [%0 + 0xa0], %%g0
+ ld [%0 + 0xb0], %%g0
+ ld [%0 + 0xc0], %%g0
+ ld [%0 + 0xd0], %%g0
+ ld [%0 + 0xe0], %%g0
+ ld [%0 + 0xf0], %%g0
+ " : : "r" (begin));
+ begin += 256;
+ }
}
+ restore_flags(flags);
}
/* Scrape the segment starting at ADDR from the virtual cache. */
@@ -161,6 +172,10 @@
{
segflushes++;
addr &= SUN4C_REAL_PGDIR_MASK;
+
+ if(sun4c_get_segmap(addr) == invalid_segment)
+ return;
+
if(sun4c_vacinfo.do_hwflushes) {
unsigned long end = (addr + sun4c_vacinfo.num_bytes);
@@ -168,6 +183,9 @@
__asm__ __volatile__("sta %%g0, [%0] %1\n\t" : :
"r" (addr), "i" (ASI_HWFLUSHSEG));
} else {
+ unsigned long nbytes = sun4c_vacinfo.num_bytes;
+ unsigned long lsize = sun4c_vacinfo.linesize;
+
__asm__ __volatile__("add %2, %2, %%g1\n\t"
"add %2, %%g1, %%g2\n\t"
"add %2, %%g2, %%g3\n\t"
@@ -177,20 +195,20 @@
"add %2, %%o4, %%o5\n"
"1:\n\t"
"subcc %1, %%o5, %1\n\t"
- "sta %%g0, [%0] %3\n\t"
- "sta %%g0, [%0 + %2] %3\n\t"
- "sta %%g0, [%0 + %%g1] %3\n\t"
- "sta %%g0, [%0 + %%g2] %3\n\t"
- "sta %%g0, [%0 + %%g3] %3\n\t"
- "sta %%g0, [%0 + %%g4] %3\n\t"
- "sta %%g0, [%0 + %%g5] %3\n\t"
- "sta %%g0, [%0 + %%o4] %3\n\t"
+ "sta %%g0, [%0] %6\n\t"
+ "sta %%g0, [%0 + %2] %6\n\t"
+ "sta %%g0, [%0 + %%g1] %6\n\t"
+ "sta %%g0, [%0 + %%g2] %6\n\t"
+ "sta %%g0, [%0 + %%g3] %6\n\t"
+ "sta %%g0, [%0 + %%g4] %6\n\t"
+ "sta %%g0, [%0 + %%g5] %6\n\t"
+ "sta %%g0, [%0 + %%o4] %6\n\t"
"bg 1b\n\t"
- " add %0, %%o5, %0\n\t" : :
- "r" (addr), "r" (sun4c_vacinfo.num_bytes),
- "r" (sun4c_vacinfo.linesize),
- "i" (ASI_FLUSHSEG) :
- "g1", "g2", "g3", "g4", "g5", "o4", "o5");
+ " add %0, %%o5, %0\n\t"
+ : "=&r" (addr), "=&r" (nbytes), "=&r" (lsize)
+ : "0" (addr), "1" (nbytes), "2" (lsize),
+ "i" (ASI_FLUSHSEG)
+ : "g1", "g2", "g3", "g4", "g5", "o4", "o5");
}
}
@@ -199,11 +217,18 @@
{
addr &= PAGE_MASK;
+ if((sun4c_get_pte(addr) & (_SUN4C_PAGE_NOCACHE | _SUN4C_PAGE_VALID)) !=
+ _SUN4C_PAGE_VALID)
+ return;
+
pageflushes++;
if(sun4c_vacinfo.do_hwflushes) {
__asm__ __volatile__("sta %%g0, [%0] %1\n\t" : :
"r" (addr), "i" (ASI_HWFLUSHPAGE));
} else {
+ unsigned long left = PAGE_SIZE;
+ unsigned long lsize = sun4c_vacinfo.linesize;
+
__asm__ __volatile__("add %2, %2, %%g1\n\t"
"add %2, %%g1, %%g2\n\t"
"add %2, %%g2, %%g3\n\t"
@@ -213,20 +238,20 @@
"add %2, %%o4, %%o5\n"
"1:\n\t"
"subcc %1, %%o5, %1\n\t"
- "sta %%g0, [%0] %3\n\t"
- "sta %%g0, [%0 + %2] %3\n\t"
- "sta %%g0, [%0 + %%g1] %3\n\t"
- "sta %%g0, [%0 + %%g2] %3\n\t"
- "sta %%g0, [%0 + %%g3] %3\n\t"
- "sta %%g0, [%0 + %%g4] %3\n\t"
- "sta %%g0, [%0 + %%g5] %3\n\t"
- "sta %%g0, [%0 + %%o4] %3\n\t"
+ "sta %%g0, [%0] %6\n\t"
+ "sta %%g0, [%0 + %2] %6\n\t"
+ "sta %%g0, [%0 + %%g1] %6\n\t"
+ "sta %%g0, [%0 + %%g2] %6\n\t"
+ "sta %%g0, [%0 + %%g3] %6\n\t"
+ "sta %%g0, [%0 + %%g4] %6\n\t"
+ "sta %%g0, [%0 + %%g5] %6\n\t"
+ "sta %%g0, [%0 + %%o4] %6\n\t"
"bg 1b\n\t"
- " add %0, %%o5, %0\n\t" : :
- "r" (addr), "r" (PAGE_SIZE),
- "r" (sun4c_vacinfo.linesize),
- "i" (ASI_FLUSHPG) :
- "g1", "g2", "g3", "g4", "g5", "o4", "o5");
+ " add %0, %%o5, %0\n\t"
+ : "=&r" (addr), "=&r" (left), "=&r" (lsize)
+ : "0" (addr), "1" (left), "2" (lsize),
+ "i" (ASI_FLUSHPG)
+ : "g1", "g2", "g3", "g4", "g5", "o4", "o5");
}
}
@@ -280,10 +305,19 @@
void sun4c_probe_vac(void)
{
sun4c_disable_vac();
- sun4c_vacinfo.num_bytes = prom_getintdefault(prom_root_node,
- "vac-size", 65536);
- sun4c_vacinfo.linesize = prom_getintdefault(prom_root_node,
- "vac-linesize", 16);
+ if((idprom->id_machtype == (SM_SUN4C | SM_4C_SS1)) ||
+ (idprom->id_machtype == (SM_SUN4C | SM_4C_SS1PLUS))) {
+ /* PROM on SS1 lacks this info, to be super safe we
+ * hard code it here since this arch is cast in stone.
+ */
+ sun4c_vacinfo.num_bytes = 65536;
+ sun4c_vacinfo.linesize = 16;
+ } else {
+ sun4c_vacinfo.num_bytes = prom_getintdefault(prom_root_node,
+ "vac-size", 65536);
+ sun4c_vacinfo.linesize = prom_getintdefault(prom_root_node,
+ "vac-linesize", 16);
+ }
sun4c_vacinfo.num_lines =
(sun4c_vacinfo.num_bytes / sun4c_vacinfo.linesize);
switch(sun4c_vacinfo.linesize) {
@@ -299,12 +333,11 @@
prom_halt();
};
- /* Only vac-hwflush (with a dash) is reliable, weitek
- * power-up processor claims vac_hwflush (underscore)
- * yet crashes if you try to use hardware based flushes.
- */
sun4c_vacinfo.do_hwflushes = prom_getintdefault(prom_root_node,
"vac-hwflush", 0);
+ if(sun4c_vacinfo.do_hwflushes == 0)
+ sun4c_vacinfo.do_hwflushes = prom_getintdefault(prom_root_node,
+ "vac_hwflush", 0);
if(sun4c_vacinfo.num_bytes != 65536) {
prom_printf("WEIRD Sun4C VAC cache size, tell davem");
@@ -320,8 +353,9 @@
extern unsigned long invalid_segment_patch2, invalid_segment_patch2_ff;
extern unsigned long num_context_patch1, num_context_patch1_16;
extern unsigned long num_context_patch2, num_context_patch2_16;
-extern unsigned long sun4c_kernel_buckets_patch;
-extern unsigned long sun4c_kernel_buckets_patch_32;
+extern unsigned long vac_linesize_patch, vac_linesize_patch_32;
+extern unsigned long vac_hwflush_patch1, vac_hwflush_patch1_on;
+extern unsigned long vac_hwflush_patch2, vac_hwflush_patch2_on;
#define PATCH_INSN(src, dst) do { \
daddr = &(dst); \
@@ -363,25 +397,38 @@
num_contexts);
prom_halt();
}
- switch (SUN4C_KERNEL_BUCKETS) {
+ if(sun4c_vacinfo.do_hwflushes != 0) {
+ PATCH_INSN(vac_hwflush_patch1_on, vac_hwflush_patch1);
+ PATCH_INSN(vac_hwflush_patch2_on, vac_hwflush_patch2);
+ } else {
+ switch(sun4c_vacinfo.linesize) {
case 16:
/* Default, nothing to do. */
break;
case 32:
- PATCH_INSN(sun4c_kernel_buckets_patch_32,
- sun4c_kernel_buckets_patch);
+ PATCH_INSN(vac_linesize_patch_32, vac_linesize_patch);
break;
default:
- prom_printf("Unhandled number of kernel buckets: %d\n",
- SUN4C_KERNEL_BUCKETS);
+ prom_printf("Impossible VAC linesize %d, halting...\n",
+ sun4c_vacinfo.linesize);
prom_halt();
+ };
}
}
static void sun4c_probe_mmu(void)
{
- num_segmaps = prom_getintdefault(prom_root_node, "mmu-npmg", 128);
- num_contexts = prom_getintdefault(prom_root_node, "mmu-nctx", 0x8);
+ if((idprom->id_machtype == (SM_SUN4C | SM_4C_SS1)) ||
+ (idprom->id_machtype == (SM_SUN4C | SM_4C_SS1PLUS))) {
+ /* Hardcode these just to be safe, PROM on SS1 does
+ * not have this info available in the root node.
+ */
+ num_segmaps = 128;
+ num_contexts = 8;
+ } else {
+ num_segmaps = prom_getintdefault(prom_root_node, "mmu-npmg", 128);
+ num_contexts = prom_getintdefault(prom_root_node, "mmu-nctx", 0x8);
+ }
patch_kernel_fault_handler();
}
@@ -408,7 +455,8 @@
extern unsigned long start;
if((idprom->id_machtype == (SM_SUN4C | SM_4C_SS2)) ||
- (idprom->id_machtype == (SM_SUN4C | SM_4C_IPX))) {
+ (idprom->id_machtype == (SM_SUN4C | SM_4C_IPX)) ||
+ (idprom->id_machtype == (SM_SUN4C | SM_4C_ELC))) {
/* Whee.. */
printk("SS2 cache bug detected, uncaching trap table page\n");
sun4c_flush_page((unsigned int) &start);
@@ -432,7 +480,8 @@
sun4c_flush_page(page);
page -= PAGE_OFFSET;
page >>= PAGE_SHIFT;
- page |= (_SUN4C_PAGE_VALID | _SUN4C_PAGE_WRITE | _SUN4C_PAGE_NOCACHE);
+ page |= (_SUN4C_PAGE_VALID | _SUN4C_PAGE_DIRTY |
+ _SUN4C_PAGE_NOCACHE | _SUN4C_PAGE_PRIV);
sun4c_put_pte(addr, page);
addr += PAGE_SIZE;
}
@@ -440,6 +489,11 @@
/* TLB management. */
+
+/* Don't change this struct without changing entry.S. This is used
+ * in the in-window kernel fault handler, and you don't want to mess
+ * with that. (See sun4c_fault in entry.S).
+ */
struct sun4c_mmu_entry {
struct sun4c_mmu_entry *next;
struct sun4c_mmu_entry *prev;
@@ -449,7 +503,7 @@
};
static struct sun4c_mmu_entry mmu_entry_pool[256];
-static void sun4c_init_mmu_entry_pool(void)
+__initfunc(static void sun4c_init_mmu_entry_pool(void))
{
int i;
@@ -500,7 +554,7 @@
}
}
-static void sun4c_init_lock_area(unsigned long start, unsigned long end)
+__initfunc(static void sun4c_init_lock_area(unsigned long start, unsigned long end))
{
int i, ctx;
@@ -516,19 +570,18 @@
}
}
+/* Don't change this struct without changing entry.S. This is used
+ * in the in-window kernel fault handler, and you don't want to mess
+ * with that. (See sun4c_fault in entry.S).
+ */
struct sun4c_mmu_ring {
struct sun4c_mmu_entry ringhd;
int num_entries;
};
static struct sun4c_mmu_ring sun4c_context_ring[16]; /* used user entries */
static struct sun4c_mmu_ring sun4c_ufree_ring; /* free user entries */
-
-static inline void sun4c_next_kernel_bucket(struct sun4c_segment_info **next)
-{
- (*next)++;
- *next = (struct sun4c_segment_info *)
- ((unsigned long)*next & ~SUN4C_KERNEL_BSIZE);
-}
+struct sun4c_mmu_ring sun4c_kernel_ring; /* used kernel entries */
+struct sun4c_mmu_ring sun4c_kfree_ring; /* free kernel entries */
static inline void sun4c_init_rings(unsigned long *mempool)
{
@@ -542,11 +595,16 @@
sun4c_ufree_ring.ringhd.next = sun4c_ufree_ring.ringhd.prev =
&sun4c_ufree_ring.ringhd;
sun4c_ufree_ring.num_entries = 0;
- /* This needs to be aligned to twice it's size for speed. */
- sun4c_kernel_next = sparc_init_alloc(mempool, 2 * SUN4C_KERNEL_BSIZE);
+ sun4c_kernel_ring.ringhd.next = sun4c_kernel_ring.ringhd.prev =
+ &sun4c_kernel_ring.ringhd;
+ sun4c_kernel_ring.num_entries = 0;
+ sun4c_kfree_ring.ringhd.next = sun4c_kfree_ring.ringhd.prev =
+ &sun4c_kfree_ring.ringhd;
+ sun4c_kfree_ring.num_entries = 0;
}
-static inline void add_ring(struct sun4c_mmu_ring *ring, struct sun4c_mmu_entry *entry)
+static inline void add_ring(struct sun4c_mmu_ring *ring,
+ struct sun4c_mmu_entry *entry)
{
struct sun4c_mmu_entry *head = &ring->ringhd;
@@ -556,7 +614,8 @@
ring->num_entries++;
}
-static inline void remove_ring(struct sun4c_mmu_ring *ring, struct sun4c_mmu_entry *entry)
+static inline void remove_ring(struct sun4c_mmu_ring *ring,
+ struct sun4c_mmu_entry *entry)
{
struct sun4c_mmu_entry *next = entry->next;
@@ -564,13 +623,15 @@
ring->num_entries--;
}
-static inline void recycle_ring(struct sun4c_mmu_ring *ring, struct sun4c_mmu_entry *entry)
+static inline void recycle_ring(struct sun4c_mmu_ring *ring,
+ struct sun4c_mmu_entry *entry)
{
struct sun4c_mmu_entry *head = &ring->ringhd;
struct sun4c_mmu_entry *next = entry->next;
(next->prev = entry->prev)->next = next;
- entry->prev = head; (entry->next = head->next)->prev = entry;
+ entry->prev = head;
+ (entry->next = head->next)->prev = entry;
head->next = entry;
/* num_entries stays the same */
}
@@ -587,7 +648,14 @@
add_ring(sun4c_context_ring+ctx, entry);
}
-static void sun4c_init_fill_kernel_ring(int howmany)
+static inline void free_kernel_entry(struct sun4c_mmu_entry *entry,
+ struct sun4c_mmu_ring *ring)
+{
+ remove_ring(ring, entry);
+ add_ring(&sun4c_kfree_ring, entry);
+}
+
+__initfunc(static void sun4c_init_fill_kernel_ring(int howmany))
{
int i;
@@ -597,14 +665,12 @@
break;
mmu_entry_pool[i].locked = 1;
sun4c_init_clean_segmap(i);
- sun4c_kernel_next->vaddr = 0;
- sun4c_kernel_next->pseg = mmu_entry_pool[i].pseg;
- sun4c_next_kernel_bucket(&sun4c_kernel_next);
+ add_ring(&sun4c_kfree_ring, &mmu_entry_pool[i]);
howmany--;
}
}
-static void sun4c_init_fill_user_ring(void)
+__initfunc(static void sun4c_init_fill_user_ring(void))
{
int i;
@@ -642,7 +708,6 @@
static inline void sun4c_user_unmap(struct sun4c_mmu_entry *uentry)
{
- /* PM: need flush_user_windows() ?? */
sun4c_put_segmap(uentry->vaddr, invalid_segment);
}
@@ -661,22 +726,25 @@
static inline void sun4c_demap_context(struct sun4c_mmu_ring *crp, unsigned char ctx)
{
struct sun4c_mmu_entry *this_entry, *next_entry;
+ unsigned long flags;
int savectx = sun4c_get_context();
+ save_and_cli(flags);
this_entry = crp->ringhd.next;
flush_user_windows();
sun4c_set_context(ctx);
- sun4c_flush_context();
while(crp->num_entries) {
next_entry = this_entry->next;
+ sun4c_flush_segment(this_entry->vaddr);
sun4c_user_unmap(this_entry);
free_user_entry(ctx, this_entry);
this_entry = next_entry;
}
sun4c_set_context(savectx);
+ restore_flags(flags);
}
-static inline void sun4c_demap_one(struct sun4c_mmu_ring *crp,unsigned char ctx)
+static inline void sun4c_demap_one(struct sun4c_mmu_ring *crp, unsigned char ctx)
{
/* by using .prev we get a kind of "lru" algorithm */
struct sun4c_mmu_entry *entry = crp->ringhd.prev;
@@ -690,6 +758,51 @@
sun4c_set_context(savectx);
}
+static int sun4c_user_taken_entries = 0;
+
+static inline struct sun4c_mmu_entry *sun4c_kernel_strategy(void)
+{
+ struct sun4c_mmu_entry *this_entry;
+
+ /* If some are free, return first one. */
+ if(sun4c_kfree_ring.num_entries) {
+ this_entry = sun4c_kfree_ring.ringhd.next;
+ return this_entry;
+ }
+
+ /* Else free one up. */
+ this_entry = sun4c_kernel_ring.ringhd.prev;
+ sun4c_flush_segment(this_entry->vaddr);
+ sun4c_kernel_unmap(this_entry);
+ free_kernel_entry(this_entry, &sun4c_kernel_ring);
+ this_entry = sun4c_kfree_ring.ringhd.next;
+
+ return this_entry;
+}
+
+void sun4c_shrink_kernel_ring(void)
+{
+ struct sun4c_mmu_entry *entry;
+ unsigned long flags;
+
+ /* If an interrupt comes in here, we die... */
+ save_and_cli(flags);
+
+ if (sun4c_user_taken_entries) {
+ entry = sun4c_kernel_strategy();
+ remove_ring(&sun4c_kfree_ring, entry);
+ add_ring(&sun4c_ufree_ring, entry);
+ sun4c_user_taken_entries--;
+#if 0
+ printk("shrink: ufree= %d, kfree= %d, kernel= %d\n",
+ sun4c_ufree_ring.num_entries,
+ sun4c_kfree_ring.num_entries,
+ sun4c_kernel_ring.num_entries);
+#endif
+ }
+ restore_flags(flags);
+}
+
/* Using this method to free up mmu entries eliminates a lot of
* potential races since we have a kernel that incurs tlb
* replacement faults. There may be performance penalties.
@@ -704,6 +817,11 @@
if(sun4c_ufree_ring.num_entries)
return sun4c_ufree_ring.ringhd.next;
+ if (sun4c_user_taken_entries) {
+ sun4c_shrink_kernel_ring();
+ return sun4c_ufree_ring.ringhd.next;
+ }
+
/* Grab one from the LRU context. */
next_one = ctx_used.next;
while (sun4c_context_ring[next_one->ctx_number].num_entries == 0)
@@ -712,10 +830,28 @@
ctx = next_one->ctx_number;
rp = &sun4c_context_ring[ctx];
- sun4c_demap_one(rp,ctx);
+ sun4c_demap_one(rp, ctx);
return sun4c_ufree_ring.ringhd.next;
}
+void sun4c_grow_kernel_ring(void)
+{
+ struct sun4c_mmu_entry *entry;
+
+ if (sun4c_ufree_ring.num_entries) {
+ entry = sun4c_ufree_ring.ringhd.next;
+ remove_ring(&sun4c_ufree_ring, entry);
+ add_ring(&sun4c_kfree_ring, entry);
+ sun4c_user_taken_entries++;
+#if 0
+ printk("grow: ufree= %d, kfree= %d, kernel= %d\n",
+ sun4c_ufree_ring.num_entries,
+ sun4c_kfree_ring.num_entries,
+ sun4c_kernel_ring.num_entries);
+#endif
+ }
+}
+
static inline void alloc_user_segment(unsigned long address, unsigned char ctx)
{
struct sun4c_mmu_entry *entry;
@@ -727,73 +863,11 @@
sun4c_user_map(entry);
}
-/* XXX Just like kernel tlb replacement we'd like to have a low level
- * XXX equivalent for user faults which need not go through the mm
- * XXX subsystem just to load a mmu entry. But this might not be as
- * XXX feasible since we need to go through the kernel page tables
- * XXX for this process, which we currently don't lock into the mmu
- * XXX so we would fault with traps off... must think about this...
- */
-void sun4c_update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t pte)
-{
- unsigned long flags;
-#if 0
- struct inode *inode;
- struct vm_area_struct *vmaring;
- unsigned long offset, vaddr;
- unsigned long start;
- pgd_t *pgdp;
- pmd_t *pmdp;
- pte_t *ptep;
-#endif
-
- save_and_cli(flags);
- address &= PAGE_MASK;
- if(sun4c_get_segmap(address) == invalid_segment)
- alloc_user_segment(address, sun4c_get_context());
- sun4c_put_pte(address, pte_val(pte));
-
-#if 0
- if (!(vma->vm_flags & VM_WRITE) ||
- !(vma->vm_flags & VM_SHARED))
- goto done;
-
- inode = vma->vm_inode;
- if (!inode)
- goto done;
-
- offset = (address & PAGE_MASK) - vma->vm_start;
- vmaring = inode->i_mmap;
- do {
- vaddr = vmaring->vm_start + offset;
-
- if (S4CVAC_BADALIAS(vaddr, address)) {
- start = vma->vm_start;
- while (start < vma->vm_end) {
- pgdp = pgd_offset(vma->vm_mm, start);
- pmdp = pmd_offset(pgdp, start);
- ptep = pte_offset(pmdp, start);
-
- if (sun4c_get_pte(start) & _SUN4C_PAGE_VALID)
- sun4c_put_pte(start, sun4c_get_pte(start) |
- _SUN4C_PAGE_NOCACHE);
-
- start += PAGE_SIZE;
- }
- goto done;
- }
- } while ((vmaring = vmaring->vm_next_share) != inode->i_mmap);
-
-done:
-#endif
- restore_flags(flags);
-}
-
/* This is now a fast in-window trap handler to avoid any and all races. */
static void sun4c_quick_kernel_fault(unsigned long address)
{
- printk("Kernel faults at addr=0x%08lx\n", address);
- panic("sun4c fault handler bolixed...");
+ printk("Kernel faults at addr 0x%08lx\n", address);
+ panic("sun4c kernel fault handler bolixed...");
}
/*
@@ -865,26 +939,35 @@
static inline void garbage_collect(int entry)
{
+ unsigned long flags;
int start, end;
+ save_and_cli(flags);
+
/* 16 buckets per segment... */
entry &= ~15;
start = entry;
for(end = (start + 16); start < end; start++)
if(sun4c_bucket[start] != BUCKET_EMPTY)
- return;
+ goto done;
/* Entire segment empty, release it. */
free_locked_segment(BUCKET_ADDR(entry));
+done:
+ restore_flags(flags);
}
static struct task_struct *sun4c_alloc_task_struct(void)
{
- unsigned long addr, page;
+ unsigned long addr, page, flags;
int entry;
+ save_and_cli(flags);
+
page = get_free_page(GFP_KERNEL);
- if(!page)
+ if(!page) {
+ restore_flags(flags);
return (struct task_struct *) 0;
+ }
/* XXX Bahh, linear search too slow, use hash
* XXX table in final implementation. Or
* XXX keep track of first free when we free
@@ -895,13 +978,21 @@
break;
if(entry == NR_TASKS) {
free_page(page);
+ restore_flags(flags);
return (struct task_struct *) 0;
}
+
+ /* Prevent an alias from occurring while the page is ours. */
+ sun4c_flush_page(page);
+
addr = BUCKET_ADDR(entry);
sun4c_bucket[entry] = (struct task_bucket *) addr;
if(sun4c_get_segmap(addr) == invalid_segment)
get_locked_segment(addr);
sun4c_put_pte(addr, BUCKET_PTE(page));
+
+ restore_flags(flags);
+
return (struct task_struct *) addr;
}
@@ -920,7 +1011,12 @@
free_page(page[0]);
return 0;
}
- saddr += (PAGE_SIZE << 1);
+
+ /* Prevent aliases from occurring while the pages are ours. */
+ sun4c_flush_page(page[0]);
+ sun4c_flush_page(page[1]);
+
+ saddr += PAGE_SIZE << 1;
sun4c_put_pte(saddr, BUCKET_PTE(page[0]));
sun4c_put_pte(saddr + PAGE_SIZE, BUCKET_PTE(page[1]));
return saddr;
@@ -929,7 +1025,9 @@
static void sun4c_free_kernel_stack(unsigned long stack)
{
unsigned long page[2];
+ unsigned long flags;
+ save_and_cli(flags);
page[0] = BUCKET_PTE_PAGE(sun4c_get_pte(stack));
page[1] = BUCKET_PTE_PAGE(sun4c_get_pte(stack+PAGE_SIZE));
sun4c_flush_page(stack);
@@ -938,22 +1036,26 @@
sun4c_put_pte(stack + PAGE_SIZE, 0);
free_page(page[0]);
free_page(page[1]);
+ restore_flags(flags);
}
static void sun4c_free_task_struct(struct task_struct *tsk)
{
unsigned long tsaddr = (unsigned long) tsk;
unsigned long page = BUCKET_PTE_PAGE(sun4c_get_pte(tsaddr));
+ unsigned long flags;
int entry = BUCKET_NUM(tsaddr);
+ save_and_cli(flags);
sun4c_flush_page(tsaddr);
sun4c_put_pte(tsaddr, 0);
sun4c_bucket[entry] = BUCKET_EMPTY;
free_page(page);
garbage_collect(entry);
+ restore_flags(flags);
}
-static void sun4c_init_buckets(void)
+__initfunc(static void sun4c_init_buckets(void))
{
int entry;
@@ -1017,7 +1119,10 @@
pte |= _SUN4C_PAGE_NOCACHE;
set_bit(scan, sun4c_iobuffer_map);
apage = (scan << PAGE_SHIFT) + sun4c_iobuffer_start;
+
+ /* Flush original mapping so we see the right things later. */
sun4c_flush_page(vpage);
+
sun4c_put_pte(apage, pte);
vpage += PAGE_SIZE;
}
@@ -1041,6 +1146,8 @@
vpage = (unsigned long)vaddr & PAGE_MASK;
npages = (((unsigned long)vaddr & ~PAGE_MASK) +
size + (PAGE_SIZE-1)) >> PAGE_SHIFT;
+
+ save_and_cli(flags);
while (npages != 0) {
--npages;
sun4c_put_pte(vpage, 0);
@@ -1050,7 +1157,6 @@
}
/* garbage collect */
- save_and_cli(flags);
scan = (sun4c_iobuffer_high - sun4c_iobuffer_start) >> PAGE_SHIFT;
while (scan >= 0 && !sun4c_iobuffer_map[scan >> 5])
scan -= 32;
@@ -1074,8 +1180,10 @@
unsigned long page;
page = ((unsigned long) bufptr) & PAGE_MASK;
- if(MAP_NR(page) > max_mapnr)
+ if(MAP_NR(page) > max_mapnr) {
+ sun4c_flush_page(page);
return bufptr; /* already locked */
+ }
return sun4c_lockarea(bufptr, len);
}
@@ -1109,7 +1217,7 @@
struct vm_area_struct sun4c_kstack_vma;
-static unsigned long sun4c_init_lock_areas(unsigned long start_mem)
+__initfunc(static unsigned long sun4c_init_lock_areas(unsigned long start_mem))
{
unsigned long sun4c_taskstack_start;
unsigned long sun4c_taskstack_end;
@@ -1135,8 +1243,6 @@
memset((void *) start_mem, 0, bitmap_size);
start_mem += bitmap_size;
- /* Now get us some mmu entries for I/O maps. */
- /* sun4c_init_lock_area(sun4c_iobuffer_start, sun4c_iobuffer_end); */
sun4c_kstack_vma.vm_mm = init_task.mm;
sun4c_kstack_vma.vm_start = sun4c_taskstack_start;
sun4c_kstack_vma.vm_end = sun4c_taskstack_end;
@@ -1149,80 +1255,99 @@
/* Cache flushing on the sun4c. */
static void sun4c_flush_cache_all(void)
{
- /* Clear all tags in the sun4c cache.
- * The cache is write through so this is safe.
- */
+ extern unsigned long bcopy;
+ unsigned long begin, end, flags;
+
+ begin = ((unsigned long) &bcopy);
+ end = (begin + sun4c_vacinfo.num_bytes);
+
+ save_and_cli(flags);
flush_user_windows();
- __asm__ __volatile__("add %2, %2, %%g1\n\t"
- "add %2, %%g1, %%g2\n\t"
- "add %2, %%g2, %%g3\n\t"
- "add %2, %%g3, %%g4\n\t"
- "add %2, %%g4, %%g5\n\t"
- "add %2, %%g5, %%o4\n\t"
- "add %2, %%o4, %%o5\n"
- "1:\n\t"
- "subcc %1, %%o5, %1\n\t"
- "sta %%g0, [%0] %3\n\t"
- "sta %%g0, [%0 + %2] %3\n\t"
- "sta %%g0, [%0 + %%g1] %3\n\t"
- "sta %%g0, [%0 + %%g2] %3\n\t"
- "sta %%g0, [%0 + %%g3] %3\n\t"
- "sta %%g0, [%0 + %%g4] %3\n\t"
- "sta %%g0, [%0 + %%g5] %3\n\t"
- "sta %%g0, [%0 + %%o4] %3\n\t"
- "bg 1b\n\t"
- " add %0, %%o5, %0\n\t" : :
- "r" (AC_CACHETAGS),
- "r" (sun4c_vacinfo.num_bytes),
- "r" (sun4c_vacinfo.linesize),
- "i" (ASI_CONTROL) :
- "g1", "g2", "g3", "g4", "g5", "o4", "o5");
+ if(sun4c_vacinfo.linesize == 32) {
+ while(begin < end) {
+ __asm__ __volatile__("
+ ld [%0 + 0x00], %%g0
+ ld [%0 + 0x20], %%g0
+ ld [%0 + 0x40], %%g0
+ ld [%0 + 0x60], %%g0
+ ld [%0 + 0x80], %%g0
+ ld [%0 + 0xa0], %%g0
+ ld [%0 + 0xc0], %%g0
+ ld [%0 + 0xe0], %%g0
+ ld [%0 + 0x100], %%g0
+ ld [%0 + 0x120], %%g0
+ ld [%0 + 0x140], %%g0
+ ld [%0 + 0x160], %%g0
+ ld [%0 + 0x180], %%g0
+ ld [%0 + 0x1a0], %%g0
+ ld [%0 + 0x1c0], %%g0
+ ld [%0 + 0x1e0], %%g0
+ " : : "r" (begin));
+ begin += 512;
+ }
+ } else {
+ while(begin < end) {
+ __asm__ __volatile__("
+ ld [%0 + 0x00], %%g0
+ ld [%0 + 0x10], %%g0
+ ld [%0 + 0x20], %%g0
+ ld [%0 + 0x30], %%g0
+ ld [%0 + 0x40], %%g0
+ ld [%0 + 0x50], %%g0
+ ld [%0 + 0x60], %%g0
+ ld [%0 + 0x70], %%g0
+ ld [%0 + 0x80], %%g0
+ ld [%0 + 0x90], %%g0
+ ld [%0 + 0xa0], %%g0
+ ld [%0 + 0xb0], %%g0
+ ld [%0 + 0xc0], %%g0
+ ld [%0 + 0xd0], %%g0
+ ld [%0 + 0xe0], %%g0
+ ld [%0 + 0xf0], %%g0
+ " : : "r" (begin));
+ begin += 256;
+ }
+ }
+ restore_flags(flags);
}
static void sun4c_flush_cache_mm(struct mm_struct *mm)
{
- int octx;
+ unsigned long flags;
+ int octx, new_ctx = mm->context;
-#ifndef __SMP__
- if(mm->context != NO_CONTEXT) {
-#endif
+ if(new_ctx != NO_CONTEXT) {
+ save_and_cli(flags);
octx = sun4c_get_context();
flush_user_windows();
- sun4c_set_context(mm->context);
+ sun4c_set_context(new_ctx);
sun4c_flush_context();
sun4c_set_context(octx);
-#ifndef __SMP__
+ restore_flags(flags);
}
-#endif
}
-
static void sun4c_flush_cache_range(struct mm_struct *mm, unsigned long start, unsigned long end)
{
- int size, size2, octx, i;
- unsigned long start2,end2;
- struct sun4c_mmu_entry *entry,*entry2;
+ int size, size2, octx, i, new_ctx = mm->context;
+ unsigned long start2, end2, flags;
+ struct sun4c_mmu_entry *entry, *entry2;
- /* don't flush kernel memory as its always valid in
- all contexts */
- if (start >= PAGE_OFFSET)
- return;
-
#if KGPROF_PROFILING
kgprof_profile();
#endif
-#ifndef __SMP__
- if(mm->context != NO_CONTEXT) {
-#endif
+ if(new_ctx != NO_CONTEXT) {
+ save_and_cli(flags);
+
size = end - start;
octx = sun4c_get_context();
flush_user_windows();
- sun4c_set_context(mm->context);
+ sun4c_set_context(new_ctx);
- entry = sun4c_context_ring[mm->context].ringhd.next;
- i = sun4c_context_ring[mm->context].num_entries;
+ entry = sun4c_context_ring[new_ctx].ringhd.next;
+ i = sun4c_context_ring[new_ctx].num_entries;
while (i--) {
entry2 = entry->next;
if (entry->vaddr < start || entry->vaddr >= end)
@@ -1241,52 +1366,55 @@
} else {
start2 &= SUN4C_REAL_PGDIR_MASK;
sun4c_flush_segment(start2);
- /* we are betting that the entry will not be
- needed for a while */
+
+ /* We are betting that the entry will not be
+ * needed for a while.
+ */
sun4c_user_unmap(entry);
- free_user_entry(mm->context, entry);
+ free_user_entry(new_ctx, entry);
}
next_entry:
entry = entry2;
}
sun4c_set_context(octx);
-#ifndef __SMP__
+ restore_flags(flags);
}
-#endif
}
static void sun4c_flush_cache_page(struct vm_area_struct *vma, unsigned long page)
{
- int octx;
+ unsigned long flags;
struct mm_struct *mm = vma->vm_mm;
-
- /* don't flush kernel memory as its always valid in
- all contexts */
- if (page >= PAGE_OFFSET)
- return;
+ int octx, new_ctx = mm->context;
/* Sun4c has no separate I/D caches so cannot optimize for non
* text page flushes.
*/
-#ifndef __SMP__
- if(mm->context != NO_CONTEXT) {
-#endif
+ if(new_ctx != NO_CONTEXT) {
+ save_and_cli(flags);
octx = sun4c_get_context();
flush_user_windows();
- sun4c_set_context(mm->context);
+ sun4c_set_context(new_ctx);
sun4c_flush_page(page);
sun4c_set_context(octx);
-#ifndef __SMP__
+ restore_flags(flags);
}
-#endif
}
-/* Sun4c cache is write-through, so no need to validate main memory
- * during a page copy in kernel space.
+/* Even though sun4c is write through, a virtual cache alias inconsistancy
+ * can still occur with COW page handling so we must flush anyways.
*/
static void sun4c_flush_page_to_ram(unsigned long page)
{
+ sun4c_flush_page(page);
+}
+
+/* Sun4c cache is unified, both instructions and data live there, so
+ * no need to flush the on-stack instructions for new signal handlers.
+ */
+static void sun4c_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr)
+{
}
/* TLB flushing on the sun4c. These routines count on the cache
@@ -1296,21 +1424,23 @@
static void sun4c_flush_tlb_all(void)
{
+ struct sun4c_mmu_entry *this_entry, *next_entry;
unsigned long flags;
- int savectx, ctx, entry;
+ int savectx, ctx;
save_and_cli(flags);
+ this_entry = sun4c_kernel_ring.ringhd.next;
savectx = sun4c_get_context();
- for (entry = 0; entry < SUN4C_KERNEL_BUCKETS; entry++) {
- if (sun4c_kernel_next->vaddr) {
- for(ctx = 0; ctx < num_contexts; ctx++) {
- sun4c_set_context(ctx);
- sun4c_put_segmap(sun4c_kernel_next->vaddr,
- invalid_segment);
- }
- sun4c_kernel_next->vaddr = 0;
+ flush_user_windows();
+ while (sun4c_kernel_ring.num_entries) {
+ next_entry = this_entry->next;
+ sun4c_flush_segment(this_entry->vaddr);
+ for(ctx = 0; ctx < num_contexts; ctx++) {
+ sun4c_set_context(ctx);
+ sun4c_put_segmap(this_entry->vaddr, invalid_segment);
}
- sun4c_next_kernel_bucket(&sun4c_kernel_next);
+ free_kernel_entry(this_entry, &sun4c_kernel_ring);
+ this_entry = next_entry;
}
sun4c_set_context(savectx);
restore_flags(flags);
@@ -1320,93 +1450,83 @@
{
struct sun4c_mmu_entry *this_entry, *next_entry;
struct sun4c_mmu_ring *crp;
- int savectx, ctx;
+ unsigned long flags;
+ int savectx, new_ctx = mm->context;
-#ifndef __SMP__
- if(mm->context != NO_CONTEXT) {
-#endif
- crp = &sun4c_context_ring[mm->context];
+ if(new_ctx != NO_CONTEXT) {
+ save_and_cli(flags);
+ crp = &sun4c_context_ring[new_ctx];
savectx = sun4c_get_context();
- ctx = mm->context;
this_entry = crp->ringhd.next;
flush_user_windows();
- sun4c_set_context(mm->context);
- sun4c_flush_context();
+ sun4c_set_context(new_ctx);
while(crp->num_entries) {
next_entry = this_entry->next;
+ sun4c_flush_segment(this_entry->vaddr);
sun4c_user_unmap(this_entry);
- free_user_entry(ctx, this_entry);
+ free_user_entry(new_ctx, this_entry);
this_entry = next_entry;
}
sun4c_set_context(savectx);
-#ifndef __SMP__
+ restore_flags(flags);
}
-#endif
}
static void sun4c_flush_tlb_range(struct mm_struct *mm, unsigned long start, unsigned long end)
{
- struct sun4c_mmu_entry *entry,*entry2;
- unsigned char savectx;
- int i;
+ struct sun4c_mmu_entry *entry, *entry2;
+ unsigned long flags;
+ int i, savectx, new_ctx = mm->context;
-#ifndef __SMP__
- if(mm->context == NO_CONTEXT)
+ if(new_ctx == NO_CONTEXT)
return;
-#endif
#if KGPROF_PROFILING
kgprof_profile();
#endif
+ save_and_cli(flags);
savectx = sun4c_get_context();
- sun4c_set_context(mm->context);
+ flush_user_windows();
+ sun4c_set_context(new_ctx);
start &= SUN4C_REAL_PGDIR_MASK;
- entry = sun4c_context_ring[mm->context].ringhd.next;
- i = sun4c_context_ring[mm->context].num_entries;
+ entry = sun4c_context_ring[new_ctx].ringhd.next;
+ i = sun4c_context_ring[new_ctx].num_entries;
while (i--) {
entry2 = entry->next;
if (entry->vaddr >= start && entry->vaddr < end) {
sun4c_flush_segment(entry->vaddr);
sun4c_user_unmap(entry);
- free_user_entry(mm->context, entry);
+ free_user_entry(new_ctx, entry);
}
entry = entry2;
}
sun4c_set_context(savectx);
+ restore_flags(flags);
}
-
static void sun4c_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
{
struct mm_struct *mm = vma->vm_mm;
- int savectx;
+ unsigned long flags;
+ int savectx, new_ctx = mm->context;
-#ifndef __SMP__
- if(mm->context != NO_CONTEXT) {
-#endif
+ if(new_ctx != NO_CONTEXT) {
+ save_and_cli(flags);
savectx = sun4c_get_context();
- sun4c_set_context(mm->context);
+ flush_user_windows();
+ sun4c_set_context(new_ctx);
page &= PAGE_MASK;
- if(sun4c_get_pte(page) & _SUN4C_PAGE_VALID)
- sun4c_put_pte(page, 0);
+ sun4c_flush_page(page);
+ sun4c_put_pte(page, 0);
sun4c_set_context(savectx);
-#ifndef __SMP__
+ restore_flags(flags);
}
-#endif
}
-/* Sun4c mmu hardware doesn't update the dirty bit in the pte's
- * for us, so we do it in software.
- */
static void sun4c_set_pte(pte_t *ptep, pte_t pte)
{
-
- if((pte_val(pte) & (_SUN4C_PAGE_WRITE|_SUN4C_PAGE_DIRTY)) ==
- _SUN4C_PAGE_WRITE)
- pte_val(pte) |= _SUN4C_PAGE_DIRTY;
-
*ptep = pte;
}
@@ -1416,17 +1536,14 @@
unsigned long page_entry;
page_entry = ((physaddr >> PAGE_SHIFT) & 0xffff);
- page_entry |= (_SUN4C_PAGE_VALID | _SUN4C_PAGE_WRITE |
- _SUN4C_PAGE_NOCACHE | _SUN4C_PAGE_IO);
+ page_entry |= ((pg_iobits | _SUN4C_PAGE_PRIV) & ~(_SUN4C_PAGE_PRESENT));
if(rdonly)
- page_entry &= (~_SUN4C_PAGE_WRITE);
- sun4c_flush_page(virt_addr);
+ page_entry &= ~_SUN4C_WRITEABLE;
sun4c_put_pte(virt_addr, page_entry);
}
void sun4c_unmapioaddr(unsigned long virt_addr)
{
- sun4c_flush_page(virt_addr); /* XXX P3: Is it necessary for I/O page? */
sun4c_put_pte(virt_addr, 0);
}
@@ -1474,7 +1591,9 @@
static void sun4c_switch_to_context(struct task_struct *tsk)
{
struct ctx_list *ctx;
+ unsigned long flags;
+ save_and_cli(flags);
if(tsk->mm->context == NO_CONTEXT) {
sun4c_alloc_context(tsk->mm);
goto set_context;
@@ -1487,13 +1606,18 @@
set_context:
sun4c_set_context(tsk->mm->context);
+ restore_flags(flags);
}
static void sun4c_flush_hook(void)
{
if(current->tss.flags & SPARC_FLAG_KTHREAD) {
+ unsigned long flags;
+
+ save_flags(flags);
sun4c_alloc_context(current->mm);
sun4c_set_context(current->mm->context);
+ restore_flags(flags);
}
}
@@ -1503,11 +1627,15 @@
struct mm_struct *mm = current->mm;
if(mm->context != NO_CONTEXT && mm->count == 1) {
+ unsigned long flags;
+
+ save_and_cli(flags);
sun4c_demap_context(&sun4c_context_ring[mm->context], mm->context);
ctx_old = ctx_list_pool + mm->context;
remove_from_ctx_list(ctx_old);
add_to_free_ctxlist(ctx_old);
mm->context = NO_CONTEXT;
+ restore_flags(flags);
}
}
@@ -1531,6 +1659,7 @@
"mmuctxs\t\t: %d\n"
"mmupsegs\t: %d\n"
"kernelpsegs\t: %d\n"
+ "kfreepsegs\t: %d\n"
"usedpsegs\t: %d\n"
"ufreepsegs\t: %d\n"
"context\t\t: %d flushes\n"
@@ -1541,8 +1670,8 @@
sun4c_vacinfo.linesize,
num_contexts,
(invalid_segment + 1),
- invalid_segment - used_user_entries -
- sun4c_ufree_ring.num_entries + 1,
+ sun4c_kernel_ring.num_entries,
+ sun4c_kfree_ring.num_entries,
used_user_entries,
sun4c_ufree_ring.num_entries,
ctxflushes, segflushes, pageflushes);
@@ -1590,17 +1719,23 @@
}
static int sun4c_pte_none(pte_t pte) { return !pte_val(pte); }
-static int sun4c_pte_present(pte_t pte) { return pte_val(pte) & _SUN4C_PAGE_VALID; }
+static int sun4c_pte_present(pte_t pte)
+{
+ return ((pte_val(pte) & (_SUN4C_PAGE_PRESENT | _SUN4C_PAGE_PRIV)) != 0);
+}
static void sun4c_pte_clear(pte_t *ptep) { pte_val(*ptep) = 0; }
static int sun4c_pmd_none(pmd_t pmd) { return !pmd_val(pmd); }
static int sun4c_pmd_bad(pmd_t pmd)
{
- return (pmd_val(pmd) & ~PAGE_MASK) != PGD_TABLE ||
- MAP_NR(pmd_val(pmd)) > max_mapnr;
+ return (((pmd_val(pmd) & ~PAGE_MASK) != PGD_TABLE) ||
+ (MAP_NR(pmd_val(pmd)) > max_mapnr));
}
-static int sun4c_pmd_present(pmd_t pmd) { return pmd_val(pmd) & PGD_PRESENT; }
+static int sun4c_pmd_present(pmd_t pmd)
+{
+ return ((pmd_val(pmd) & PGD_PRESENT) != 0);
+}
static void sun4c_pmd_clear(pmd_t *pmdp) { pmd_val(*pmdp) = 0; }
static int sun4c_pgd_none(pgd_t pgd) { return 0; }
@@ -1612,16 +1747,62 @@
* The following only work if pte_present() is true.
* Undefined behaviour if not..
*/
-static int sun4c_pte_write(pte_t pte) { return pte_val(pte) & _SUN4C_PAGE_WRITE; }
-static int sun4c_pte_dirty(pte_t pte) { return pte_val(pte) & _SUN4C_PAGE_DIRTY; }
-static int sun4c_pte_young(pte_t pte) { return pte_val(pte) & _SUN4C_PAGE_REF; }
-
-static pte_t sun4c_pte_wrprotect(pte_t pte) { pte_val(pte) &= ~_SUN4C_PAGE_WRITE; return pte; }
-static pte_t sun4c_pte_mkclean(pte_t pte) { pte_val(pte) &= ~_SUN4C_PAGE_DIRTY; return pte; }
-static pte_t sun4c_pte_mkold(pte_t pte) { pte_val(pte) &= ~_SUN4C_PAGE_REF; return pte; }
-static pte_t sun4c_pte_mkwrite(pte_t pte) { pte_val(pte) |= _SUN4C_PAGE_WRITE; return pte; }
-static pte_t sun4c_pte_mkdirty(pte_t pte) { pte_val(pte) |= _SUN4C_PAGE_DIRTY; return pte; }
-static pte_t sun4c_pte_mkyoung(pte_t pte) { pte_val(pte) |= _SUN4C_PAGE_REF; return pte; }
+static int sun4c_pte_write(pte_t pte)
+{
+ return pte_val(pte) & _SUN4C_PAGE_WRITE;
+}
+
+static int sun4c_pte_dirty(pte_t pte)
+{
+ return pte_val(pte) & _SUN4C_PAGE_MODIFIED;
+}
+
+static int sun4c_pte_young(pte_t pte)
+{
+ return pte_val(pte) & _SUN4C_PAGE_ACCESSED;
+}
+
+static pte_t sun4c_pte_wrprotect(pte_t pte)
+{
+ pte_val(pte) &= ~(_SUN4C_PAGE_WRITE | _SUN4C_PAGE_SILENT_WRITE);
+ return pte;
+}
+
+static pte_t sun4c_pte_mkclean(pte_t pte)
+{
+ pte_val(pte) &= ~(_SUN4C_PAGE_MODIFIED | _SUN4C_PAGE_SILENT_WRITE);
+ return pte;
+}
+
+static pte_t sun4c_pte_mkold(pte_t pte)
+{
+ pte_val(pte) &= ~(_SUN4C_PAGE_ACCESSED | _SUN4C_PAGE_SILENT_READ);
+ return pte;
+}
+
+static pte_t sun4c_pte_mkwrite(pte_t pte)
+{
+ pte_val(pte) |= _SUN4C_PAGE_WRITE;
+ if (pte_val(pte) & _SUN4C_PAGE_MODIFIED)
+ pte_val(pte) |= _SUN4C_PAGE_SILENT_WRITE;
+ return pte;
+}
+
+static pte_t sun4c_pte_mkdirty(pte_t pte)
+{
+ pte_val(pte) |= _SUN4C_PAGE_MODIFIED;
+ if (pte_val(pte) & _SUN4C_PAGE_WRITE)
+ pte_val(pte) |= _SUN4C_PAGE_SILENT_WRITE;
+ return pte;
+}
+
+static pte_t sun4c_pte_mkyoung(pte_t pte)
+{
+ pte_val(pte) |= _SUN4C_PAGE_ACCESSED;
+ if (pte_val(pte) & _SUN4C_PAGE_READ)
+ pte_val(pte) |= _SUN4C_PAGE_SILENT_READ;
+ return pte;
+}
/*
* Conversion functions: convert a page and protection to a page entry,
@@ -1644,7 +1825,9 @@
static pte_t sun4c_pte_modify(pte_t pte, pgprot_t newprot)
{
- return __pte((pte_val(pte) & _SUN4C_PAGE_CHG_MASK) | pgprot_val(newprot));
+ pte_val(pte) = (pte_val(pte) & _SUN4C_PAGE_CHG_MASK) |
+ pgprot_val(newprot);
+ return pte;
}
static unsigned long sun4c_pte_page(pte_t pte)
@@ -1680,13 +1863,35 @@
{
}
+/* Please take special note on the foo_kernel() routines below, our
+ * fast in window fault handler wants to get at the pte's for vmalloc
+ * area with traps off, therefore they _MUST_ be locked down to prevent
+ * a watchdog from happening. The lowest VMALLOC'd thing is the buffer
+ * cache hash table, it is never free'd ever, so the pg0 we use for the
+ * lowest most page of pte's will never be freed and is guarenteed to
+ * be locked down. Be careful with this code thanks.
+ */
+
/* Allocate and free page tables. The xxx_kernel() versions are
* used to allocate a kernel page table - this turns on ASN bits
* if any, and marks the page tables reserved.
*/
static void sun4c_pte_free_kernel(pte_t *pte)
{
- free_page((unsigned long) pte);
+ unsigned long pg0_addr = (unsigned long) pg0;
+ unsigned long this_addr = (unsigned long) pte;
+ unsigned long page = (unsigned long) pte;
+
+ pg0_addr &= PAGE_MASK;
+ this_addr &= PAGE_MASK;
+
+ /* See above comment. */
+ if(pg0_addr != this_addr) {
+ page = (PAGE_OFFSET + ((sun4c_get_pte(this_addr) & 0xffff)<<PAGE_SHIFT));
+ sun4c_unlockarea((char *)this_addr, PAGE_SIZE);
+ }
+
+ free_page(page);
}
static pte_t *sun4c_pte_alloc_kernel(pmd_t *pmd, unsigned long address)
@@ -1696,6 +1901,8 @@
pte_t *page = (pte_t *) get_free_page(GFP_KERNEL);
if (sun4c_pmd_none(*pmd)) {
if (page) {
+ /* See above comment. */
+ page = (pte_t *) sun4c_lockarea((char *)page, PAGE_SIZE);
pmd_val(*pmd) = PGD_TABLE | (unsigned long) page;
return page + address;
}
@@ -1778,11 +1985,95 @@
return (pgd_t *) get_free_page(GFP_KERNEL);
}
+/* There are really two cases of aliases to watch out for, and these
+ * are:
+ *
+ * 1) A user's page which can be aliased with the kernels virtual
+ * mapping of the physical page.
+ *
+ * 2) Multiple user mappings of the same inode/anonymous object
+ * such that two copies of the same data for the same phys page
+ * can live (writable) in the cache at the same time.
+ *
+ * We handle number 1 by flushing the kernel copy of the page always
+ * after COW page operations.
+ *
+ * NOTE: We are a bit slowed down now because the VMA arg is indeed used
+ * now, so our ref/mod bit tracking quick userfaults eat a few more
+ * cycles than they used to.
+ */
+void sun4c_update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t pte)
+{
+ unsigned long offset, vaddr, start, flags;
+ pgd_t *pgdp;
+ pte_t *ptep;
+
+ save_and_cli(flags);
+ address &= PAGE_MASK;
+ if(sun4c_get_segmap(address) == invalid_segment)
+ alloc_user_segment(address, sun4c_get_context());
+
+ if((vma->vm_flags & (VM_WRITE|VM_SHARED)) == (VM_WRITE|VM_SHARED)) {
+ struct vm_area_struct *vmaring;
+ struct inode *inode;
+ int alias_found = 0;
+
+ /* While on the other hand, this is very uncommon... */
+ inode = vma->vm_inode;
+ if(!inode)
+ goto done;
+
+ offset = (address & PAGE_MASK) - vma->vm_start;
+ vmaring = inode->i_mmap;
+ do {
+ vaddr = vmaring->vm_start + offset;
+
+ if (S4CVAC_BADALIAS(vaddr, address)) {
+ alias_found++;
+ start = vmaring->vm_start;
+ while(start < vmaring->vm_end) {
+ pgdp = sun4c_pgd_offset(vmaring->vm_mm, start);
+ if(!pgdp) goto next;
+ ptep = sun4c_pte_offset((pmd_t *) pgdp, start);
+ if(!ptep) goto next;
+
+ if(pte_val(*ptep) & _SUN4C_PAGE_PRESENT) {
+#if 1
+ printk("Fixing USER/USER alias [%d:%08lx]\n",
+ vmaring->vm_mm->context, start);
+#endif
+ sun4c_flush_cache_page(vmaring, start);
+ pte_val(*ptep) = (pte_val(*ptep) |
+ _SUN4C_PAGE_NOCACHE);
+ sun4c_flush_tlb_page(vmaring, start);
+ }
+ next:
+ start += PAGE_SIZE;
+ }
+ }
+ } while ((vmaring = vmaring->vm_next_share) != inode->i_mmap);
+
+ if(alias_found && !(pte_val(pte) & _SUN4C_PAGE_NOCACHE)) {
+ pgdp = sun4c_pgd_offset(vma->vm_mm, address);
+ ptep = sun4c_pte_offset((pmd_t *) pgdp, address);
+ pte_val(*ptep) = (pte_val(*ptep) | _SUN4C_PAGE_NOCACHE);
+ pte = pte_val(*ptep);
+ }
+ }
+done:
+ sun4c_put_pte(address, pte_val(pte));
+ restore_flags(flags);
+}
+
+static void sun4c_pgd_flush(pgd_t *pgdp)
+{
+}
+
extern unsigned long free_area_init(unsigned long, unsigned long);
extern unsigned long sparc_context_init(unsigned long, int);
extern unsigned long end;
-unsigned long sun4c_paging_init(unsigned long start_mem, unsigned long end_mem)
+__initfunc(unsigned long sun4c_paging_init(unsigned long start_mem, unsigned long end_mem))
{
int i, cnt;
unsigned long kernel_end;
@@ -1811,7 +2102,6 @@
PGD_TABLE | (unsigned long) pg0;
sun4c_init_ss2_cache_bug();
start_mem = PAGE_ALIGN(start_mem);
- /* start_mem = sun4c_init_alloc_dvma_pages(start_mem); */
start_mem = sparc_context_init(start_mem, num_contexts);
start_mem = free_area_init(start_mem, end_mem);
cnt = 0;
@@ -1823,7 +2113,7 @@
}
/* Load up routines and constants for sun4c mmu */
-void ld_mmu_sun4c(void)
+__initfunc(void ld_mmu_sun4c(void))
{
printk("Loading sun4c MMU routines\n");
@@ -1844,11 +2134,10 @@
page_copy = SUN4C_PAGE_COPY;
page_readonly = SUN4C_PAGE_READONLY;
page_kernel = SUN4C_PAGE_KERNEL;
- pg_iobits = _SUN4C_PAGE_NOCACHE | _SUN4C_PAGE_IO | _SUN4C_PAGE_VALID
- | _SUN4C_PAGE_WRITE | _SUN4C_PAGE_DIRTY;
+ pg_iobits = _SUN4C_PAGE_PRESENT | _SUN4C_READABLE | _SUN4C_WRITEABLE |
+ _SUN4C_PAGE_IO | _SUN4C_PAGE_NOCACHE;
/* Functions */
-#ifndef __SMP__
flush_cache_all = sun4c_flush_cache_all;
flush_cache_mm = sun4c_flush_cache_mm;
flush_cache_range = sun4c_flush_cache_range;
@@ -1858,30 +2147,11 @@
flush_tlb_mm = sun4c_flush_tlb_mm;
flush_tlb_range = sun4c_flush_tlb_range;
flush_tlb_page = sun4c_flush_tlb_page;
-#else
- local_flush_cache_all = sun4c_flush_cache_all;
- local_flush_cache_mm = sun4c_flush_cache_mm;
- local_flush_cache_range = sun4c_flush_cache_range;
- local_flush_cache_page = sun4c_flush_cache_page;
-
- local_flush_tlb_all = sun4c_flush_tlb_all;
- local_flush_tlb_mm = sun4c_flush_tlb_mm;
- local_flush_tlb_range = sun4c_flush_tlb_range;
- local_flush_tlb_page = sun4c_flush_tlb_page;
-
- flush_cache_all = smp_flush_cache_all;
- flush_cache_mm = smp_flush_cache_mm;
- flush_cache_range = smp_flush_cache_range;
- flush_cache_page = smp_flush_cache_page;
-
- flush_tlb_all = smp_flush_tlb_all;
- flush_tlb_mm = smp_flush_tlb_mm;
- flush_tlb_range = smp_flush_tlb_range;
- flush_tlb_page = smp_flush_tlb_page;
-#endif
flush_page_to_ram = sun4c_flush_page_to_ram;
+ flush_sig_insns = sun4c_flush_sig_insns;
+
set_pte = sun4c_set_pte;
switch_to_context = sun4c_switch_to_context;
pmd_align = sun4c_pmd_align;
@@ -1924,6 +2194,7 @@
pmd_alloc = sun4c_pmd_alloc;
pgd_free = sun4c_pgd_free;
pgd_alloc = sun4c_pgd_alloc;
+ pgd_flush = sun4c_pgd_flush;
pte_write = sun4c_pte_write;
pte_dirty = sun4c_pte_dirty;
FUNET's LINUX-ADM group, linux-adm@nic.funet.fi
TCL-scripts by Sam Shen, slshen@lbl.gov