Subject: Cache flushing, take 2

From: "Justin Carlson" <[email-masked]>
Date: 30 May 2002 12:28:03 -0700

I've taken a second look at the cache flushing, and think maybe this is
a better way to fix what I see as the major problems.

Attached is a patch which does the following:

* Removes __flush_cache_all()
* Adds writeback_inv_dcache_all() and writeback_inv_dcache_range().
  these functions force the flushing of the dcache, as opposed to the
  flush_cache_* functions, which only flush if needed for coherence.
* Adds better documentation to the function declarations in pgtable.h
* Fixes up gdb-stub.c to use this interface
* Actually implements the more agressive semantics for the cacheflush 
  system call
* Fixes up the old sysmips system call to use this interface

I've only implemented the actual new routines for the sb1; I'd like to
solicit some feedback as to what other people think of this approach
before taking the plunge to other implementations.  Note this patch is
not tested beyond compilability; if people like this approach I'll flesh
it out and resubmit something tested.

Comments?

-Justin

? cacheflush.patch
Index: arch/mips/kernel/gdb-stub.c
===================================================================
RCS file: /cvs/linux/arch/mips/kernel/gdb-stub.c,v
retrieving revision 1.16
diff -u -r1.16 gdb-stub.c
--- arch/mips/kernel/gdb-stub.c	2002/05/29 22:40:04	1.16
+++ arch/mips/kernel/gdb-stub.c	2002/05/30 19:13:10
@@ -575,7 +575,7 @@
 	async_bp.addr = epc;
 	async_bp.val  = *(unsigned *)epc;
 	*(unsigned *)epc = BP;
-	__flush_cache_all();
+	flush_icache_range(epc, epc + 4);
 }

@@ -799,10 +799,8 @@
 			 * has no way of knowing that a data ref to some location
 			 * may have changed something that is in the instruction
 			 * cache.
-			 * NB: We flush both caches, just to be sure...
 			 */
-
-			__flush_cache_all();
+			flush_icache_all();
 			return;
 			/* NOTREACHED */
 			break;
@@ -831,7 +829,7 @@
 			 * use breakpoints and continue, instead.
 			 */
 			single_step(regs);
-			__flush_cache_all();
+			flush_icache_all();
 			return;
 			/* NOTREACHED */
 
Index: arch/mips/kernel/sysmips.c
===================================================================
RCS file: /cvs/linux/arch/mips/kernel/sysmips.c,v
retrieving revision 1.21
diff -u -r1.21 sysmips.c
--- arch/mips/kernel/sysmips.c	2002/05/29 18:36:28	1.21
+++ arch/mips/kernel/sysmips.c	2002/05/30 19:13:10
@@ -83,7 +83,8 @@
 		goto out;
 
 	case FLUSH_CACHE:
-		__flush_cache_all();
+		flush_icache_all();
+		writeback_inv_dcache_all();
 		retval = 0;
 		goto out;
 
Index: arch/mips/mm/c-sb1.c
===================================================================
RCS file: /cvs/linux/arch/mips/mm/c-sb1.c,v
retrieving revision 1.19
diff -u -r1.19 c-sb1.c
--- arch/mips/mm/c-sb1.c	2002/05/29 22:40:05	1.19
+++ arch/mips/mm/c-sb1.c	2002/05/30 19:13:10
@@ -74,72 +74,6 @@
 {
 }
 
-static void local_sb1___flush_cache_all(void)
-{
-	/*
-	 * Haven't worried too much about speed here; given that we're flushing
-	 * the icache, the time to invalidate is dwarfed by the time it's going
-	 * to take to refill it.  Register usage:
-	 * 
-	 * $1 - moving cache index
-	 * $2 - set count
-	 */
-	__asm__ __volatile__ (
-		".set push                  \n"
-		".set noreorder             \n"
-		".set noat                  \n"
-		".set mips4                 \n"
-		"     move   $1, %2         \n" /* Start at index 0 */
-		"1:   cache  %3, 0($1)      \n" /* WB/Invalidate this index */
-		"     addiu  %1, %1, -1     \n" /* Decrement loop count */
-		"     bnez   %1, 1b         \n" /* loop test */
-		"      addu   $1, $1, %0    \n" /* Next address */
-		".set pop                   \n"
-		:
-		: "r" (dcache_line_size), "r" (dcache_sets * dcache_assoc),
-		  "r" (KSEG0), "i" (Index_Writeback_Inv_D));
-
-	__asm__ __volatile__ (
-		".set push                  \n"
-		".set noreorder             \n"
-		".set mips2                 \n"
-		"sync                       \n"
-#ifdef CONFIG_SB1_PASS_1_WORKAROUNDS		/* Bug 1384 */			
-		"sync                       \n"
-#endif
-		".set pop                   \n");
-
-	__asm__ __volatile__ (
-		".set push                  \n"
-		".set noreorder             \n"
-		".set noat                  \n"
-		".set mips4                 \n"
-		"     move   $1, %2         \n" /* Start at index 0 */
-		"1:   cache  %3, 0($1)       \n" /* Invalidate this index */
-		"     addiu  %1, %1, -1     \n" /* Decrement loop count */
-		"     bnez   %1, 1b         \n" /* loop test */
-		"      addu   $1, $1, %0    \n" /* Next address */
-		".set pop                   \n"
-		:
-		: "r" (icache_line_size), "r" (icache_sets * icache_assoc),
-		  "r" (KSEG0), "i" (Index_Invalidate_I));
-}
-
-#ifdef CONFIG_SMP
-extern void sb1___flush_cache_all_ipi(void *ignored);
-asm("sb1___flush_cache_all_ipi = local_sb1___flush_cache_all");
-
-static void sb1___flush_cache_all(void)
-{
-	smp_call_function(sb1___flush_cache_all_ipi, 0, 1, 1);
-	local_sb1___flush_cache_all();
-}
-#else
-extern void sb1___flush_cache_all(void);
-asm("sb1___flush_cache_all = local_sb1___flush_cache_all");
-#endif
-
-
 /*
  * When flushing a range in the icache, we have to first writeback
  * the dcache for the same range, so new ifetches will see any
@@ -216,33 +150,129 @@
 }
 
 #ifdef CONFIG_SMP
-struct flush_icache_range_args {
+
+struct flush_cache_range_args {
 	unsigned long start;
 	unsigned long end;
 };
 
 static void sb1_flush_icache_range_ipi(void *info)
 {
-	struct flush_icache_range_args *args = info;
+	struct flush_cache_range_args *args = info;
 
 	local_sb1_flush_icache_range(args->start, args->end);
 }
 
-void sb1_flush_icache_range(unsigned long start, unsigned long end)
+void smp_sb1_flush_icache_range(unsigned long start, unsigned long end)
 {
-	struct flush_icache_range_args args;
+	struct flush_cache_range_args args;
 
 	args.start = start;
 	args.end = end;
 	smp_call_function(sb1_flush_icache_range_ipi, &args, 1, 1);
 	local_sb1_flush_icache_range(start, end);
 }
-#else
-void sb1_flush_icache_range(unsigned long start, unsigned long end);
-asm("sb1_flush_icache_range = local_sb1_flush_icache_range");
 #endif
 
 /*
+ * Writeback and invalidate a range of addresses.
+ */
+static void local_sb1_writeback_inv_dcache_range(unsigned long start, unsigned long end)
+{
+#ifdef CONFIG_SB1_PASS_1_WORKAROUNDS
+	unsigned long flags;
+	local_irq_save(flags);
+#endif
+
+	__asm__ __volatile__ (
+		".set push                  \n"
+		".set noreorder             \n"
+		".set noat                  \n"
+		".set mips4                 \n"
+		"     move   $1, %0         \n" 
+		"1:                         \n"
+#ifdef CONFIG_SB1_PASS_1_WORKAROUNDS
+		".align 3                   \n"
+		"     lw     $0,   0($1)    \n" /* Bug 1370, 1368            */
+		"     sync                  \n"
+#endif
+		"     cache  %3, 0($1)      \n" /* Hit-WB{,-inval} this address */
+		"     bne    $1, %1, 1b     \n" /* loop test */
+		"      addu  $1, $1, %2     \n" /* next line */
+		"     sync                  \n"
+		".set pop                   \n"
+		:
+		: "r" (start  & ~(dcache_line_size - 1)),
+		  "r" ((end - 1) & ~(dcache_line_size - 1)),
+		  "r" (dcache_line_size),
+		  "i" (Hit_Writeback_Inv_D)
+		);
+#ifdef CONFIG_SB1_PASS_1_WORKAROUNDS
+	local_irq_restore(flags);
+#endif
+}
+
+
+#ifdef CONFIG_SMP
+
+static void sb1_writeback_inv_dcache_range_ipi(void *info)
+{
+	struct flush_cache_range_args *args = info;
+
+	local_sb1_writeback_inv_dcache_range(args->start, args->end);
+}
+
+void smp_sb1_writeback_inv_dcache_range(unsigned long start, unsigned long end)
+{
+	struct flush_cache_range_args args;
+
+	args.start = start;
+	args.end = end;
+	smp_call_function(sb1_writeback_inv_dcache_range_ipi, &args, 1, 1);
+	local_sb1_flush_icache_range(start, end);
+}
+
+#endif
+
+/*
+ * Writeback and invalidate the entire dcache
+ */
+static void local_sb1_writeback_inv_dcache_all(void)
+{
+	/*
+	 * Register usage:
+	 * 
+	 * $1 - moving cache index
+	 * $2 - set count
+	 */
+	__asm__ __volatile__ (
+		".set push                  \n"
+		".set noreorder             \n"
+		".set noat                  \n"
+		".set mips4                 \n"
+		"     move   $1, %2         \n" /* Start at index 0 */
+		"1:   cache  %3, 0($1)      \n" /* Invalidate this index */
+		"     addiu  %1, %1, -1     \n" /* Decrement loop count */
+		"     bnez   %1, 1b         \n" /* loop test */
+		"      addu   $1, $1, %0    \n" /* Next address */
+		".set pop                   \n"
+		:
+		: "r" (dcache_line_size), "r" (dcache_sets * dcache_assoc),
+		  "r" (KSEG0), "i" (Index_Writeback_Inv_D));
+}
+
+
+#ifdef CONFIG_SMP
+
+void smp_sb1_writeback_inv_dcache_all(void)
+{
+	smp_call_function((void (*)(void *))local_sb1_writeback_inv_dcache_all, 0, 1, 1);
+	local_sb1_writeback_inv_dcache_all();
+}
+
+#endif
+
+/*
  * If there's no context yet, or the page isn't executable, no icache flush
  * is needed
  */
@@ -257,7 +287,7 @@
 	 * conservatively flush the entire caches on all processors
 	 * (ouch).
 	 */
-	sb1___flush_cache_all();
+	flush_icache_all();
 }
 
 static inline void protected_flush_icache_line(unsigned long addr)
@@ -345,7 +375,7 @@
 	protected_flush_icache_line(iaddr);
 }
 
-static void sb1_flush_cache_sigtramp(unsigned long addr)
+static void smp_sb1_flush_cache_sigtramp(unsigned long addr)
 {
 	unsigned long tmp;
 
@@ -359,10 +389,6 @@
 
 	smp_call_function(sb1_flush_cache_sigtramp_ipi, (void *) addr, 1, 1);
 }
-
-#else
-void sb1_flush_cache_sigtramp(unsigned long addr);
-asm("sb1_flush_cache_sigtramp = local_sb1_flush_cache_sigtramp");
 #endif
 
 static void sb1_flush_icache_all(void)
@@ -506,17 +532,26 @@
 	_copy_page = sb1_copy_page;
 
 	_flush_cache_all = sb1_flush_cache_all;
-	___flush_cache_all = sb1___flush_cache_all;
 	_flush_cache_mm = (void (*)(struct mm_struct *))sb1_nop;
 	_flush_cache_range = (void *) sb1_nop;
 	_flush_page_to_ram = sb1_flush_page_to_ram;
 	_flush_icache_page = sb1_flush_icache_page;
-	_flush_icache_range = sb1_flush_icache_range;
+
+#ifdef CONFIG_SMP
+	_writeback_inv_dcache_range = smp_sb1_writeback_inv_dcache_range;
+	_writeback_inv_dcache_all   = smp_sb1_writeback_inv_dcache_all;
+	_flush_icache_range         = smp_sb1_flush_icache_range;
+	_flush_cache_sigtramp       = smp_sb1_flush_cache_sigtramp;
+#else
+	_writeback_inv_dcache_range = local_sb1_writeback_inv_dcache_range;
+	_writeback_inv_dcache_all   = local_sb1_writeback_inv_dcache_all;
+	_flush_icache_range         = local_sb1_flush_icache_range;
+	_flush_cache_sigtramp       = local_sb1_flush_cache_sigtramp;	
+#endif
 
 	/* None of these are needed for the sb1 */
 	_flush_cache_page = (void *) sb1_nop;
 
-	_flush_cache_sigtramp = sb1_flush_cache_sigtramp;
 	_flush_icache_all = sb1_flush_icache_all;
 
 	change_cp0_config(CONF_CM_CMASK, CONF_CM_CACHABLE_COW);
Index: arch/mips/mm/init.c
===================================================================
RCS file: /cvs/linux/arch/mips/mm/init.c,v
retrieving revision 1.43
diff -u -r1.43 init.c
--- arch/mips/mm/init.c	2002/03/15 03:14:31	1.43
+++ arch/mips/mm/init.c	2002/05/30 19:13:10
@@ -37,6 +37,7 @@
 #include <asm/pgalloc.h>
 #include <asm/mmu_context.h>
 #include <asm/tlb.h>
+#include <asm/uaccess.h>
 
 mmu_gather_t mmu_gathers[NR_CPUS];
 unsigned long highstart_pfn, highend_pfn;
@@ -48,8 +49,21 @@
 
 asmlinkage int sys_cacheflush(void *addr, int bytes, int cache)
 {
-	/* This should flush more selectivly ...  */
-	__flush_cache_all();
+	if (cache & ~(ICACHE | DCACHE)) {
+		return -EINVAL;
+	}
+	
+	if (verify_area(VERIFY_READ, addr, bytes)) {
+		return -EFAULT;
+	}
+
+	if (cache & DCACHE) {
+		writeback_inv_dcache_range((unsigned long)addr, (unsigned long)addr + bytes);
+	} 
+
+	if (cache & ICACHE) {
+		flush_icache_range((unsigned long)addr, ((unsigned long)addr) + bytes);
+	}
 
 	return 0;
 }
Index: arch/mips/mm/loadmmu.c
===================================================================
RCS file: /cvs/linux/arch/mips/mm/loadmmu.c,v
retrieving revision 1.45
diff -u -r1.45 loadmmu.c
--- arch/mips/mm/loadmmu.c	2001/11/29 04:47:24	1.45
+++ arch/mips/mm/loadmmu.c	2002/05/30 19:13:10
@@ -24,7 +24,6 @@
 
 /* Cache operations. */
 void (*_flush_cache_all)(void);
-void (*___flush_cache_all)(void);
 void (*_flush_cache_mm)(struct mm_struct *mm);
 void (*_flush_cache_range)(struct mm_struct *mm, unsigned long start,
 			  unsigned long end);
@@ -35,6 +34,8 @@
 
 void (*_flush_page_to_ram)(struct page * page);
 void (*_flush_icache_all)(void);
+void (*_writeback_inv_dcache_range)(unsigned long start, unsigned long end);
+void (*_writeback_inv_dcache_all)  (void);
 
 #ifdef CONFIG_NONCOHERENT_IO
 
Index: include/asm-mips/pgtable.h
===================================================================
RCS file: /cvs/linux/include/asm-mips/pgtable.h,v
retrieving revision 1.74
diff -u -r1.74 pgtable.h
--- include/asm-mips/pgtable.h	2002/05/28 09:58:58	1.74
+++ include/asm-mips/pgtable.h	2002/05/30 19:13:22
@@ -17,49 +17,70 @@
 #include <asm/cachectl.h>
 #include <asm/fixmap.h>
 
-/* Cache flushing:
+/* Generic cache flushing.  See Documentation/cachetlb for more specific details
  *
- *  - flush_cache_all() flushes entire cache
- *  - flush_cache_mm(mm) flushes the specified mm context's cache lines
- *  - flush_cache_page(mm, vmaddr) flushes a single page
- *  - flush_cache_range(mm, start, end) flushes a range of pages
- *  - flush_page_to_ram(page) write back kernel page to ram
- *  - flush_icache_range(start, end) flush a range of instructions
+ * Note that none of these routines check access permissions.  Any needed
+ * checking should be done by the caller.
  *
- *  - flush_cache_sigtramp() flush signal trampoline
- *  - flush_icache_all() flush the entire instruction cache
+ *  - flush_cache_all      Writeback & inval all virtually mapped cached data
+ *                           & ensure all writes are visible to the system.
+ *  - flush_cache_mm       Same as above, but only for a specific mm context
+ *  - flush_cache_page     Same as above, but for a single page
+ *  - flush_cache_range    Same as above, for a range of addresses
+ *  - flush_page_to_ram    Clear all virtual mappings of this physical page
+ *  - flush_icache_range   An istream modification may have occurred; ensure 
+ *                           that all data stores before this point are visible 
+ *                           for new instruction fetches for the given range.
+ *  - flush_icache_page    Same as above, but for a specific page
  */
-extern void (*_flush_cache_all)(void);
-extern void (*___flush_cache_all)(void);
-extern void (*_flush_cache_mm)(struct mm_struct *mm);
-extern void (*_flush_cache_range)(struct mm_struct *mm, unsigned long start,
-				 unsigned long end);
-extern void (*_flush_cache_page)(struct vm_area_struct *vma, unsigned long page);
-extern void (*_flush_page_to_ram)(struct page * page);
+
+/*
+ * Mips-specific cache flushing.  
+ *  - flush_icache_all           Same as flush_icache_range, for all memory 
+ *  - writeback_inv_dcache_all   Force writeback and invalidation of the first
+ *                               level dcache.
+ *  - writeback_inv_dcache_range Same as above, but for a specific virtual range
+ *  - flush_cache_sigtramp       Flush signal trampoline
+ */
+
+extern void (*_flush_cache_all)   (void);
+extern void (*_flush_cache_mm)    (struct mm_struct *mm);
+extern void (*_flush_cache_page)  (struct vm_area_struct *vma, 
+				   unsigned long page);
+extern void (*_flush_cache_range) (struct mm_struct *mm, unsigned long start,
+				   unsigned long end);
+extern void (*_flush_page_to_ram) (struct page * page);
 extern void (*_flush_icache_range)(unsigned long start, unsigned long end);
-extern void (*_flush_icache_page)(struct vm_area_struct *vma,
-                                  struct page *page);
-extern void (*_flush_cache_sigtramp)(unsigned long addr);
-extern void (*_flush_icache_all)(void);
+extern void (*_flush_icache_page) (struct vm_area_struct *vma,
+				   struct page *page);
+
+extern void (*_flush_icache_all)          (void);
+extern void (*_writeback_inv_dcache_all)  (void);
+extern void (*_writeback_inv_dcache_range)(unsigned long start, 
+					   unsigned long end);
+extern void (*_flush_cache_sigtramp)      (unsigned long addr);
+
 
 #define flush_dcache_page(page)			do { } while (0)
 
 #define flush_cache_all()		_flush_cache_all()
-#define __flush_cache_all()		___flush_cache_all()
 #define flush_cache_mm(mm)		_flush_cache_mm(mm)
-#define flush_cache_range(mm,start,end)	_flush_cache_range(mm,start,end)
 #define flush_cache_page(vma,page)	_flush_cache_page(vma, page)
+#define flush_cache_range(mm,start,end)	_flush_cache_range(mm,start,end)
 #define flush_page_to_ram(page)		_flush_page_to_ram(page)
-
 #define flush_icache_range(start, end)	_flush_icache_range(start,end)
 #define flush_icache_page(vma, page) 	_flush_icache_page(vma, page)
+
+
 
-#define flush_cache_sigtramp(addr)	_flush_cache_sigtramp(addr)
 #ifdef CONFIG_VTAG_ICACHE
-#define flush_icache_all()		_flush_icache_all()
+#define flush_icache_all()	    	       _flush_icache_all()
 #else
-#define flush_icache_all()		do { } while(0)
+#define flush_icache_all()		       do { } while(0)
 #endif
+#define writeback_inv_dcache_all()             _writeback_inv_dcache_all()
+#define writeback_inv_dcache_range(start, end) _writeback_inv_dcache_range(start, end)
+#define flush_cache_sigtramp(addr)	       _flush_cache_sigtramp(addr)
 
 /*
  * - add_wired_entry() add a fixed TLB entry, and move wired register