patch-2.1.101 linux/arch/mips/mm/r4xx0.c

• Lines: 771
• Date: Fri May 8 00:13:24 1998
• Orig file: v2.1.100/linux/arch/mips/mm/r4xx0.c
• Orig date: Wed Dec 10 10:31:10 1997

diff -u --recursive --new-file v2.1.100/linux/arch/mips/mm/r4xx0.c linux/arch/mips/mm/r4xx0.c
@@ -3,7 +3,7 @@
  *
  * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
  *
- * $Id: r4xx0.c,v 1.11 1997/12/02 06:33:53 ralf Exp $
+ * $Id: r4xx0.c,v 1.11 1998/05/04 09:18:31 ralf Exp $
  *
  * To do:
  *
@@ -11,13 +11,13 @@
  *  - many of the bug workarounds are not efficient at all, but at
  *    least they are functional ...
  */
-#include <linux/config.h>
-
+#include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/sched.h>
 #include <linux/mm.h>
-#include <linux/autoconf.h>
 
+#include <asm/bcache.h>
+#include <asm/io.h>
 #include <asm/sgi.h>
 #include <asm/sgimc.h>
 #include <asm/page.h>
@@ -45,6 +45,18 @@
 #undef DEBUG_CACHE
 
 /*
+ * Dummy cache handling routines for machines without boardcaches
+ */
+static void no_sc_noop(void) {}
+
+static struct bcache_ops no_sc_ops = {
+	(void *)no_sc_noop, (void *)no_sc_noop,
+	(void *)no_sc_noop, (void *)no_sc_noop
+};
+
+struct bcache_ops *bcops = &no_sc_ops;
+
+/*
  * On processors with QED R4600 style two set assosicative cache
  * this is the bit which selects the way in the cache for the
  * indexed cachops.
@@ -53,9 +65,19 @@
 #define dcache_waybit (dcache_size >> 1)
 
 /*
- * Zero an entire page.  We have three flavours of the routine available.
- * One for CPU with 16byte, with 32byte cachelines plus a special version
- * with nops which handles the buggy R4600 v1.x.
+ * Zero an entire page.  Basically a simple unrolled loop should do the
+ * job but we want more performance by saving memory bus bandwidth.  We
+ * have five flavours of the routine available for:
+ *
+ * - 16byte cachelines and no second level cache
+ * - 32byte cachelines second level cache
+ * - a version which handles the buggy R4600 v1.x
+ * - a version which handles the buggy R4600 v2.0
+ * - Finally a last version without fancy cache games for the SC and MC
+ *   versions of R4000 and R4400.  Cache instructions are quite expensive
+ *   and I guess using them for both the primary and the second level cache
+ *   wouldn't be worth the effort.
+ *   This needs to be verified by benchmarking.
  */
 
 static void r4k_clear_page_d16(unsigned long page)
@@ -219,6 +241,32 @@
 	restore_flags(flags);
 }
 
+static void r4k_clear_page(unsigned long page)
+{
+	__asm__ __volatile__(
+		".set\tnoreorder\n\t"
+		".set\tnoat\n\t"
+		".set\tmips3\n\t"
+		"daddiu\t$1,%0,%2\n"
+		"1:\tsd\t$0,(%0)\n\t"
+		"sd\t$0,8(%0)\n\t"
+		"sd\t$0,16(%0)\n\t"
+		"sd\t$0,24(%0)\n\t"
+		"daddiu\t%0,64\n\t"
+		"sd\t$0,-32(%0)\n\t"
+		"sd\t$0,-24(%0)\n\t"
+		"sd\t$0,-16(%0)\n\t"
+		"bne\t$1,%0,1b\n\t"
+		"sd\t$0,-8(%0)\n\t"
+		".set\tmips0\n\t"
+		".set\tat\n\t"
+		".set\treorder"
+		:"=r" (page)
+		:"0" (page),
+		 "I" (PAGE_SIZE)
+		:"$1","memory");
+}
+
 
 /*
  * This is still inefficient.  We only can do better if we know the
@@ -477,6 +525,60 @@
 	restore_flags(flags);
 }
 
+static void r4k_copy_page(unsigned long to, unsigned long from)
+{
+	unsigned long dummy1, dummy2;
+	unsigned long reg1, reg2, reg3, reg4;
+
+	__asm__ __volatile__(
+		".set\tnoreorder\n\t"
+		".set\tnoat\n\t"
+		".set\tmips3\n\t"
+		"daddiu\t$1,%0,%8\n"
+		"1:\tlw\t%2,(%1)\n\t"
+		"lw\t%3,4(%1)\n\t"
+		"lw\t%4,8(%1)\n\t"
+		"lw\t%5,12(%1)\n\t"
+		"sw\t%2,(%0)\n\t"
+		"sw\t%3,4(%0)\n\t"
+		"sw\t%4,8(%0)\n\t"
+		"sw\t%5,12(%0)\n\t"
+		"lw\t%2,16(%1)\n\t"
+		"lw\t%3,20(%1)\n\t"
+		"lw\t%4,24(%1)\n\t"
+		"lw\t%5,28(%1)\n\t"
+		"sw\t%2,16(%0)\n\t"
+		"sw\t%3,20(%0)\n\t"
+		"sw\t%4,24(%0)\n\t"
+		"sw\t%5,28(%0)\n\t"
+		"daddiu\t%0,64\n\t"
+		"daddiu\t%1,64\n\t"
+		"lw\t%2,-32(%1)\n\t"
+		"lw\t%3,-28(%1)\n\t"
+		"lw\t%4,-24(%1)\n\t"
+		"lw\t%5,-20(%1)\n\t"
+		"sw\t%2,-32(%0)\n\t"
+		"sw\t%3,-28(%0)\n\t"
+		"sw\t%4,-24(%0)\n\t"
+		"sw\t%5,-20(%0)\n\t"
+		"lw\t%2,-16(%1)\n\t"
+		"lw\t%3,-12(%1)\n\t"
+		"lw\t%4,-8(%1)\n\t"
+		"lw\t%5,-4(%1)\n\t"
+		"sw\t%2,-16(%0)\n\t"
+		"sw\t%3,-12(%0)\n\t"
+		"sw\t%4,-8(%0)\n\t"
+		"bne\t$1,%0,1b\n\t"
+		"sw\t%5,-4(%0)\n\t"
+		".set\tmips0\n\t"
+		".set\tat\n\t"
+		".set\treorder"
+		:"=r" (dummy1), "=r" (dummy2),
+		 "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4)
+		:"0" (to), "1" (from),
+		 "I" (PAGE_SIZE));
+}
+
 /*
  * If you think for one second that this stuff coming up is a lot
  * of bulky code eating too many kernel cache lines.  Think _again_.
@@ -1892,144 +1994,111 @@
  * in .pdf format.)
  */
 static void
-r4k_flush_cache_pre_dma_out_pc(unsigned long addr, unsigned long size)
+r4k_dma_cache_wback_inv_pc(unsigned long addr, unsigned long size)
 {
 	unsigned long end, a;
-	unsigned int cmode, flags;
-
-	cmode = read_32bit_cp0_register(CP0_CONFIG) & CONFIG_CM_CMASK;
-	if (cmode == CONFIG_CM_CACHABLE_WA ||
-	    cmode == CONFIG_CM_CACHABLE_NO_WA) {
-		/* primary dcache is writethrough, therefore memory
-		   is already consistent with the caches.  */
-		return;
-	}
+	unsigned int flags;
 
 	if (size >= dcache_size) {
 		flush_cache_all();
-		return;
-	}
-
-	/* Workaround for R4600 bug.  See comment above. */
-	save_and_cli(flags);
-	*(volatile unsigned long *)KSEG1;
+	} else {
+		/* Workaround for R4600 bug.  See comment above. */
+		save_and_cli(flags);
+		*(volatile unsigned long *)KSEG1;
 
-	a = addr & ~(dc_lsize - 1);
-	end = (addr + size) & ~(dc_lsize - 1);
-	while (1) {
-		flush_dcache_line(a); /* Hit_Writeback_Inv_D */
-		if (a == end) break;
-		a += dc_lsize;
+		a = addr & ~(dc_lsize - 1);
+		end = (addr + size) & ~(dc_lsize - 1);
+		while (1) {
+			flush_dcache_line(a); /* Hit_Writeback_Inv_D */
+			if (a == end) break;
+			a += dc_lsize;
+		}
+		restore_flags(flags);
 	}
-	restore_flags(flags);
+	bcops->bc_wback_inv(addr, size);
 }
 
 static void
-r4k_flush_cache_pre_dma_out_sc(unsigned long addr, unsigned long size)
+r4k_dma_cache_wback_inv_sc(unsigned long addr, unsigned long size)
 {
-	unsigned long end;
-	unsigned long a;
+	unsigned long end, a;
+	unsigned int flags;
 
 	if (size >= scache_size) {
 		flush_cache_all();
-		return;
+	} else {
+		save_and_cli(flags);
+		a = addr & ~(dc_lsize - 1);
+		end = (addr + size) & ~(dc_lsize - 1);
+		while (1) {
+			flush_dcache_line(a); /* Hit_Writeback_Inv_D */
+			if (a == end) break;
+			a += dc_lsize;
+		}
+		restore_flags(flags);
 	}
 
 	a = addr & ~(sc_lsize - 1);
 	end = (addr + size) & ~(sc_lsize - 1);
 	while (1) {
-		flush_scache_line(addr); /* Hit_Writeback_Inv_SD */
-		if (addr == end) break;
-		addr += sc_lsize;
+		flush_scache_line(a);	/* Hit_Writeback_Inv_SD */
+		if (a == end) break;
+		a += sc_lsize;
 	}
-	r4k_flush_cache_pre_dma_out_pc(addr, size);
 }
 
 static void
-r4k_flush_cache_post_dma_in_pc(unsigned long addr, unsigned long size)
+r4k_dma_cache_inv_pc(unsigned long addr, unsigned long size)
 {
-	unsigned long end;
-	unsigned long a;
+	unsigned long end, a;
+	unsigned int flags;
 
 	if (size >= dcache_size) {
 		flush_cache_all();
-		return;
-	}
-
-	/* Workaround for R4600 bug.  See comment above. */
-	*(volatile unsigned long *)KSEG1;
+	} else {
+		/* Workaround for R4600 bug.  See comment above. */
+		save_and_cli(flags);
+		*(volatile unsigned long *)KSEG1;
 
-	a = addr & ~(dc_lsize - 1);
-	end = (addr + size) & ~(dc_lsize - 1);
-	while (1) {
-		invalidate_dcache_line(a); /* Hit_Invalidate_D */
-		if (a == end) break;
-		a += dc_lsize;
+		a = addr & ~(dc_lsize - 1);
+		end = (addr + size) & ~(dc_lsize - 1);
+		while (1) {
+			flush_dcache_line(a); /* Hit_Writeback_Inv_D */
+			if (a == end) break;
+			a += dc_lsize;
+		}
+		restore_flags(flags);
 	}
+
+	bcops->bc_inv(addr, size);
 }
 
 static void
-r4k_flush_cache_post_dma_in_sc(unsigned long addr, unsigned long size)
+r4k_dma_cache_inv_sc(unsigned long addr, unsigned long size)
 {
-	unsigned long end;
-	unsigned long a;
+	unsigned long end, a;
+	unsigned int flags;
 
 	if (size >= scache_size) {
 		flush_cache_all();
-		return;
+	} else {
+		save_and_cli(flags);
+		a = addr & ~(dc_lsize - 1);
+		end = (addr + size) & ~(dc_lsize - 1);
+		while (1) {
+			flush_dcache_line(a); /* Hit_Writeback_Inv_D */
+			if (a == end) break;
+			a += dc_lsize;
+		}
+		restore_flags(flags);
 	}
 
 	a = addr & ~(sc_lsize - 1);
 	end = (addr + size) & ~(sc_lsize - 1);
 	while (1) {
-		invalidate_scache_line(addr); /* Hit_Invalidate_SD */
-		if (addr == end) break;
-		addr += sc_lsize;
-	}
-	r4k_flush_cache_pre_dma_out_pc(addr, size);
-}
-
-static void r4k_flush_page_to_ram_d32i32_r4600(unsigned long page)
-{
-	page &= PAGE_MASK;
-	if((page >= KSEG0 && page < KSEG1) || (page >= KSEG2)) {
-		unsigned long flags;
-
-#ifdef DEBUG_CACHE
-		printk("r4600_cram[%08lx]", page);
-#endif
-		save_and_cli(flags);
-		blast_dcache32_page(page);
-#ifdef CONFIG_SGI
-		{
-			unsigned long tmp1, tmp2;
-
-			__asm__ __volatile__("
-			.set noreorder
-			.set mips3
-			li	%0, 0x1
-			dsll	%0, 31
-			or	%0, %0, %2
-			lui	%1, 0x9000
-			dsll32	%1, 0
-			or	%0, %0, %1
-			daddu	%1, %0, 0x0fe0
-			li	%2, 0x80
-			mtc0	%2, $12
-			nop; nop; nop; nop;
-1:			sw	$0, 0(%0)
-			bltu	%0, %1, 1b
-			daddu	%0, 32
-			mtc0	$0, $12
-			nop; nop; nop; nop;
-			.set mips0
-			.set reorder"
-			: "=&r" (tmp1), "=&r" (tmp2),
-			  "=&r" (page)
-			: "2" (page & 0x0007f000));
-		}
-#endif /* CONFIG_SGI */
-		restore_flags(flags);
+		flush_scache_line(a); /* Hit_Writeback_Inv_SD */
+		if (a == end) break;
+		a += sc_lsize;
 	}
 }
 
@@ -2394,173 +2463,31 @@
 }
 
 /* Detect and size the various r4k caches. */
-static void probe_icache(unsigned long config)
+__initfunc(static void probe_icache(unsigned long config))
 {
-	unsigned long tmp;
-
-	tmp = (config >> 9) & 7;
-	icache_size = (1 << (12 + tmp));
-	if((config >> 5) & 1)
-		ic_lsize = 32;
-	else
-		ic_lsize = 16;
+	icache_size = 1 << (12 + ((config >> 6) & 7));
+	ic_lsize = 16 << ((config >> 4) & 1);
 
-	printk("Primary ICACHE %dK (linesize %d bytes)\n",
-	       (int)(icache_size >> 10), (int)ic_lsize);
+	printk("Primary instruction cache %dkb, linesize %d bytes)\n",
+	       icache_size >> 10, ic_lsize);
 }
 
-static void probe_dcache(unsigned long config)
+__initfunc(static void probe_dcache(unsigned long config))
 {
-	unsigned long tmp;
+	dcache_size = 1 << (12 + ((config >> 6) & 7));
+	dc_lsize = 16 << ((config >> 4) & 1);
 
-	tmp = (config >> 6) & 7;
-	dcache_size = (1 << (12 + tmp));
-	if((config >> 4) & 1)
-		dc_lsize = 32;
-	else
-		dc_lsize = 16;
-
-	printk("Primary DCACHE %dK (linesize %d bytes)\n",
-	       (int)(dcache_size >> 10), (int)dc_lsize);
+	printk("Primary data cache %dkb, linesize %d bytes)\n",
+	       dcache_size >> 10, dc_lsize);
 }
 
-static int probe_scache_eeprom(unsigned long config)
-{
-#ifdef CONFIG_SGI
-	volatile unsigned int *cpu_control;
-	unsigned short cmd = 0xc220;
-	unsigned long data = 0;
-	int i, n;
-
-#ifdef __MIPSEB__
-	cpu_control = (volatile unsigned int *) KSEG1ADDR(0x1fa00034);
-#else
-	cpu_control = (volatile unsigned int *) KSEG1ADDR(0x1fa00030);
-#endif
-#define DEASSERT(bit) (*(cpu_control) &= (~(bit)))
-#define ASSERT(bit) (*(cpu_control) |= (bit))
-#define DELAY  for(n = 0; n < 100000; n++) __asm__ __volatile__("")
-	DEASSERT(SGIMC_EEPROM_PRE);
-	DEASSERT(SGIMC_EEPROM_SDATAO);
-	DEASSERT(SGIMC_EEPROM_SECLOCK);
-	DEASSERT(SGIMC_EEPROM_PRE);
-	DELAY;
-	ASSERT(SGIMC_EEPROM_CSEL); ASSERT(SGIMC_EEPROM_SECLOCK);
-	for(i = 0; i < 11; i++) {
-		if(cmd & (1<<15))
-			ASSERT(SGIMC_EEPROM_SDATAO);
-		else
-			DEASSERT(SGIMC_EEPROM_SDATAO);
-		DEASSERT(SGIMC_EEPROM_SECLOCK);
-		ASSERT(SGIMC_EEPROM_SECLOCK);
-		cmd <<= 1;
-	}
-	DEASSERT(SGIMC_EEPROM_SDATAO);
-	for(i = 0; i < (sizeof(unsigned short) * 8); i++) {
-		unsigned int tmp;
-
-		DEASSERT(SGIMC_EEPROM_SECLOCK);
-		DELAY;
-		ASSERT(SGIMC_EEPROM_SECLOCK);
-		DELAY;
-		data <<= 1;
-		tmp = *cpu_control;
-		if(tmp & SGIMC_EEPROM_SDATAI)
-			data |= 1;
-	}
-	DEASSERT(SGIMC_EEPROM_SECLOCK);
-	DEASSERT(SGIMC_EEPROM_CSEL);
-	ASSERT(SGIMC_EEPROM_PRE);
-	ASSERT(SGIMC_EEPROM_SECLOCK);
-	data <<= PAGE_SHIFT;
-	printk("R4600/R5000 SCACHE size %dK ", (int) (data >> 10));
-	switch(mips_cputype) {
-	case CPU_R4600:
-	case CPU_R4640:
-		sc_lsize = 32;
-		break;
-
-	default:
-		sc_lsize = 128;
-		break;
-	}
-	printk("linesize %d bytes\n", sc_lsize);
-	scache_size = data;
-	if(data) {
-		unsigned long addr, tmp1, tmp2;
-
-		/* Enable r4600/r5000 cache.  But flush it first. */
-		for(addr = KSEG0; addr < (KSEG0 + dcache_size);
-		    addr += dc_lsize)
-			flush_dcache_line_indexed(addr);
-		for(addr = KSEG0; addr < (KSEG0 + icache_size);
-		    addr += ic_lsize)
-			flush_icache_line_indexed(addr);
-		for(addr = KSEG0; addr < (KSEG0 + scache_size);
-		    addr += sc_lsize)
-			flush_scache_line_indexed(addr);
-
-		/* R5000 scache enable is in CP0 config, on R4600 variants
-		 * the scache is enable by the memory mapped cache controller.
-		 */
-		if(mips_cputype == CPU_R5000) {
-			unsigned long config;
-
-			config = read_32bit_cp0_register(CP0_CONFIG);
-			config |= 0x1000;
-			write_32bit_cp0_register(CP0_CONFIG, config);
-		} else {
-			/* This is really cool... */
-			printk("Enabling R4600 SCACHE\n");
-			__asm__ __volatile__("
-			.set noreorder
-			.set mips3
-			mfc0	%2, $12
-			nop; nop; nop; nop;
-			li	%1, 0x80
-			mtc0	%1, $12
-			nop; nop; nop; nop;
-			li	%0, 0x1
-			dsll	%0, 31
-			lui	%1, 0x9000
-			dsll32	%1, 0
-			or	%0, %1, %0
-			sb	$0, 0(%0)
-			mtc0	$0, $12
-			nop; nop; nop; nop;
-			mtc0	%2, $12
-			nop; nop; nop; nop;
-			.set mips0
-			.set reorder
-		        " : "=r" (tmp1), "=r" (tmp2), "=r" (addr));
-		}
-
-		return 1;
-	} else {
-		if(mips_cputype == CPU_R5000)
-			return -1;
-		else
-			return 0;
-	}
-#else
-	/*
-	 * XXX For now we don't panic and assume that existing chipset
-	 * controlled caches are setup correnctly and are completly
-	 * transparent.  Works fine for those MIPS machines I know.
-	 * Morituri the salutant ...
-	 */
-	return 0;
-
-	panic("Cannot probe SCACHE on this machine.");
-#endif
-}
 
 /* If you even _breathe_ on this function, look at the gcc output
  * and make sure it does not pop things on and off the stack for
  * the cache sizing loop that executes in KSEG1 space or else
  * you will crash and burn badly.  You have been warned.
  */
-static int probe_scache(unsigned long config)
+__initfunc(static int probe_scache(unsigned long config))
 {
 	extern unsigned long stext;
 	unsigned long flags, addr, begin, end, pow2;
@@ -2644,7 +2571,7 @@
 	return 1;
 }
 
-static void setup_noscache_funcs(void)
+__initfunc(static void setup_noscache_funcs(void))
 {
 	unsigned int prid;
 
@@ -2677,18 +2604,16 @@
 		flush_page_to_ram = r4k_flush_page_to_ram_d32i32;
 		break;
 	}
-	flush_cache_pre_dma_out = r4k_flush_cache_pre_dma_out_pc;
-	flush_cache_post_dma_in = r4k_flush_cache_post_dma_in_pc;
+	dma_cache_wback_inv = r4k_dma_cache_wback_inv_pc;
+	dma_cache_inv = r4k_dma_cache_inv_pc;
 }
 
-static void setup_scache_funcs(void)
+__initfunc(static void setup_scache_funcs(void))
 {
 	switch(sc_lsize) {
 	case 16:
 		switch(dc_lsize) {
 		case 16:
-			clear_page = r4k_clear_page_d16;
-			copy_page = r4k_copy_page_d16;
 			flush_cache_all = r4k_flush_cache_all_s16d16i16;
 			flush_cache_mm = r4k_flush_cache_mm_s16d16i16;
 			flush_cache_range = r4k_flush_cache_range_s16d16i16;
@@ -2696,8 +2621,6 @@
 			flush_page_to_ram = r4k_flush_page_to_ram_s16d16i16;
 			break;
 		case 32:
-			clear_page = r4k_clear_page_d32;
-			copy_page = r4k_copy_page_d32;
 			flush_cache_all = r4k_flush_cache_all_s16d32i32;
 			flush_cache_mm = r4k_flush_cache_mm_s16d32i32;
 			flush_cache_range = r4k_flush_cache_range_s16d32i32;
@@ -2709,8 +2632,6 @@
 	case 32:
 		switch(dc_lsize) {
 		case 16:
-			clear_page = r4k_clear_page_d16;
-			copy_page = r4k_copy_page_d16;
 			flush_cache_all = r4k_flush_cache_all_s32d16i16;
 			flush_cache_mm = r4k_flush_cache_mm_s32d16i16;
 			flush_cache_range = r4k_flush_cache_range_s32d16i16;
@@ -2718,8 +2639,6 @@
 			flush_page_to_ram = r4k_flush_page_to_ram_s32d16i16;
 			break;
 		case 32:
-			clear_page = r4k_clear_page_d32;
-			copy_page = r4k_copy_page_d32;
 			flush_cache_all = r4k_flush_cache_all_s32d32i32;
 			flush_cache_mm = r4k_flush_cache_mm_s32d32i32;
 			flush_cache_range = r4k_flush_cache_range_s32d32i32;
@@ -2730,8 +2649,6 @@
 	case 64:
 		switch(dc_lsize) {
 		case 16:
-			clear_page = r4k_clear_page_d16;
-			copy_page = r4k_copy_page_d16;
 			flush_cache_all = r4k_flush_cache_all_s64d16i16;
 			flush_cache_mm = r4k_flush_cache_mm_s64d16i16;
 			flush_cache_range = r4k_flush_cache_range_s64d16i16;
@@ -2739,8 +2656,6 @@
 			flush_page_to_ram = r4k_flush_page_to_ram_s64d16i16;
 			break;
 		case 32:
-			clear_page = r4k_clear_page_d32;
-			copy_page = r4k_copy_page_d32;
 			flush_cache_all = r4k_flush_cache_all_s64d32i32;
 			flush_cache_mm = r4k_flush_cache_mm_s64d32i32;
 			flush_cache_range = r4k_flush_cache_range_s64d32i32;
@@ -2751,8 +2666,6 @@
 	case 128:
 		switch(dc_lsize) {
 		case 16:
-			clear_page = r4k_clear_page_d16;
-			copy_page = r4k_copy_page_d16;
 			flush_cache_all = r4k_flush_cache_all_s128d16i16;
 			flush_cache_mm = r4k_flush_cache_mm_s128d16i16;
 			flush_cache_range = r4k_flush_cache_range_s128d16i16;
@@ -2760,8 +2673,6 @@
 			flush_page_to_ram = r4k_flush_page_to_ram_s128d16i16;
 			break;
 		case 32:
-			clear_page = r4k_clear_page_d32;
-			copy_page = r4k_copy_page_d32;
 			flush_cache_all = r4k_flush_cache_all_s128d32i32;
 			flush_cache_mm = r4k_flush_cache_mm_s128d32i32;
 			flush_cache_range = r4k_flush_cache_range_s128d32i32;
@@ -2771,77 +2682,57 @@
 		};
 		break;
 	}
-
-	/* XXX Do these for Indy style caches also.  No need for now ... */
-	flush_cache_pre_dma_out = r4k_flush_cache_pre_dma_out_sc;
-	flush_cache_post_dma_in = r4k_flush_cache_post_dma_in_sc;
+	clear_page = r4k_clear_page;
+	copy_page = r4k_copy_page;
+	dma_cache_wback_inv = r4k_dma_cache_wback_inv_sc;
+	dma_cache_inv = r4k_dma_cache_inv_sc;
 }
 
 typedef int (*probe_func_t)(unsigned long);
-static probe_func_t probe_scache_kseg1;
 
-void ld_mmu_r4xx0(void)
+__initfunc(static inline void setup_scache(unsigned int config))
 {
-	unsigned long cfg = read_32bit_cp0_register(CP0_CONFIG);
+	probe_func_t probe_scache_kseg1;
 	int sc_present = 0;
 
+	/* Maybe the cpu knows about a l2 cache? */
+	probe_scache_kseg1 = (probe_func_t) (KSEG1ADDR(&probe_scache));
+	sc_present = probe_scache_kseg1(config);
+
+	if (sc_present) {
+		setup_scache_funcs();
+		return;
+	}
+
+	setup_noscache_funcs();
+}
+
+static int r4k_user_mode(struct pt_regs *regs)
+{
+	return (regs->cp0_status & ST0_KSU) == KSU_USER;
+}
+
+
+__initfunc(void ld_mmu_r4xx0(void))
+{
+	unsigned long config = read_32bit_cp0_register(CP0_CONFIG);
+
 	printk("CPU revision is: %08x\n", read_32bit_cp0_register(CP0_PRID));
 
 	set_cp0_config(CONFIG_CM_CMASK, CONFIG_CM_CACHABLE_NONCOHERENT);
-//set_cp0_config(CONFIG_CM_CMASK, CONFIG_CM_CACHABLE_WA);
-//set_cp0_config(CONFIG_CM_CMASK, CONFIG_CM_CACHABLE_NO_WA);
 
-	probe_icache(cfg);
-	probe_dcache(cfg);
+	probe_icache(config);
+	probe_dcache(config);
+	setup_scache(config);
 
 	switch(mips_cputype) {
-	case CPU_R4000PC:
-	case CPU_R4000SC:
-	case CPU_R4000MC:
-	case CPU_R4400PC:
-	case CPU_R4400SC:
-	case CPU_R4400MC:
-		probe_scache_kseg1 = (probe_func_t) (KSEG1ADDR(&probe_scache));
-		sc_present = probe_scache_kseg1(cfg);
-		break;
-
-	case CPU_R4600:
-	case CPU_R4640:
+	case CPU_R4600:			/* QED style two way caches? */
 	case CPU_R4700:
-	case CPU_R5000:	/* XXX: We don't handle the true R5000 SCACHE */
+	case CPU_R5000:
 	case CPU_NEVADA:
-		probe_scache_kseg1 = (probe_func_t)
-			(KSEG1ADDR(&probe_scache_eeprom));
-		sc_present = probe_scache_eeprom(cfg);
-
-		/* Try using tags if eeprom gives us bogus data. */
-		if(sc_present == -1) {
-			probe_scache_kseg1 =
-				(probe_func_t) (KSEG1ADDR(&probe_scache));
-			sc_present = probe_scache_kseg1(cfg);
-		}
-		break;
-	};
-
-	if(sc_present == 1
-	   && (mips_cputype == CPU_R4000SC
-               || mips_cputype == CPU_R4000MC
-               || mips_cputype == CPU_R4400SC
-               || mips_cputype == CPU_R4400MC)) {
-		/* Has a true secondary cache. */
-		setup_scache_funcs();
-	} else {
-		/* Lacks true secondary cache. */
-		setup_noscache_funcs();
-		if((mips_cputype != CPU_R5000)) { /* XXX */
-			flush_cache_page =
-				r4k_flush_cache_page_d32i32_r4600;
-			flush_page_to_ram =
-				r4k_flush_page_to_ram_d32i32_r4600;
-		}
+		flush_cache_page = r4k_flush_cache_page_d32i32_r4600;
 	}
 
-	/* XXX Handle true second level cache w/ split I/D */
 	flush_cache_sigtramp = r4k_flush_cache_sigtramp;
 	if ((read_32bit_cp0_register(CP0_PRID) & 0xfff0) == 0x2020) {
 		flush_cache_sigtramp = r4600v20k_flush_cache_sigtramp;
@@ -2859,6 +2750,8 @@
 	show_regs = r4k_show_regs;
     
         add_wired_entry = r4k_add_wired_entry;
+
+	user_mode = r4k_user_mode;
 
 	flush_cache_all();
 	write_32bit_cp0_register(CP0_WIRED, 0);