I've taken a second look at the cache flushing, and think maybe this is
a better way to fix what I see as the major problems.
Attached is a patch which does the following:
* Removes __flush_cache_all()
* Adds writeback_inv_dcache_all() and writeback_inv_dcache_range().
these functions force the flushing of the dcache, as opposed to the
flush_cache_* functions, which only flush if needed for coherence.
* Adds better documentation to the function declarations in pgtable.h
* Fixes up gdb-stub.c to use this interface
* Actually implements the more agressive semantics for the cacheflush
system call
* Fixes up the old sysmips system call to use this interface
I've only implemented the actual new routines for the sb1; I'd like to
solicit some feedback as to what other people think of this approach
before taking the plunge to other implementations. Note this patch is
not tested beyond compilability; if people like this approach I'll flesh
it out and resubmit something tested.
Comments?
-Justin
? cacheflush.patch
Index: arch/mips/kernel/gdb-stub.c
===================================================================
RCS file: /cvs/linux/arch/mips/kernel/gdb-stub.c,v
retrieving revision 1.16
diff -u -r1.16 gdb-stub.c
--- arch/mips/kernel/gdb-stub.c 2002/05/29 22:40:04 1.16
+++ arch/mips/kernel/gdb-stub.c 2002/05/30 19:13:10
@@ -575,7 +575,7 @@
async_bp.addr = epc;
async_bp.val = *(unsigned *)epc;
*(unsigned *)epc = BP;
- __flush_cache_all();
+ flush_icache_range(epc, epc + 4);
}
@@ -799,10 +799,8 @@
* has no way of knowing that a data ref to some location
* may have changed something that is in the instruction
* cache.
- * NB: We flush both caches, just to be sure...
*/
-
- __flush_cache_all();
+ flush_icache_all();
return;
/* NOTREACHED */
break;
@@ -831,7 +829,7 @@
* use breakpoints and continue, instead.
*/
single_step(regs);
- __flush_cache_all();
+ flush_icache_all();
return;
/* NOTREACHED */
Index: arch/mips/kernel/sysmips.c
===================================================================
RCS file: /cvs/linux/arch/mips/kernel/sysmips.c,v
retrieving revision 1.21
diff -u -r1.21 sysmips.c
--- arch/mips/kernel/sysmips.c 2002/05/29 18:36:28 1.21
+++ arch/mips/kernel/sysmips.c 2002/05/30 19:13:10
@@ -83,7 +83,8 @@
goto out;
case FLUSH_CACHE:
- __flush_cache_all();
+ flush_icache_all();
+ writeback_inv_dcache_all();
retval = 0;
goto out;
Index: arch/mips/mm/c-sb1.c
===================================================================
RCS file: /cvs/linux/arch/mips/mm/c-sb1.c,v
retrieving revision 1.19
diff -u -r1.19 c-sb1.c
--- arch/mips/mm/c-sb1.c 2002/05/29 22:40:05 1.19
+++ arch/mips/mm/c-sb1.c 2002/05/30 19:13:10
@@ -74,72 +74,6 @@
{
}
-static void local_sb1___flush_cache_all(void)
-{
- /*
- * Haven't worried too much about speed here; given that we're flushing
- * the icache, the time to invalidate is dwarfed by the time it's going
- * to take to refill it. Register usage:
- *
- * $1 - moving cache index
- * $2 - set count
- */
- __asm__ __volatile__ (
- ".set push \n"
- ".set noreorder \n"
- ".set noat \n"
- ".set mips4 \n"
- " move $1, %2 \n" /* Start at index 0 */
- "1: cache %3, 0($1) \n" /* WB/Invalidate this index */
- " addiu %1, %1, -1 \n" /* Decrement loop count */
- " bnez %1, 1b \n" /* loop test */
- " addu $1, $1, %0 \n" /* Next address */
- ".set pop \n"
- :
- : "r" (dcache_line_size), "r" (dcache_sets * dcache_assoc),
- "r" (KSEG0), "i" (Index_Writeback_Inv_D));
-
- __asm__ __volatile__ (
- ".set push \n"
- ".set noreorder \n"
- ".set mips2 \n"
- "sync \n"
-#ifdef CONFIG_SB1_PASS_1_WORKAROUNDS /* Bug 1384 */
- "sync \n"
-#endif
- ".set pop \n");
-
- __asm__ __volatile__ (
- ".set push \n"
- ".set noreorder \n"
- ".set noat \n"
- ".set mips4 \n"
- " move $1, %2 \n" /* Start at index 0 */
- "1: cache %3, 0($1) \n" /* Invalidate this index */
- " addiu %1, %1, -1 \n" /* Decrement loop count */
- " bnez %1, 1b \n" /* loop test */
- " addu $1, $1, %0 \n" /* Next address */
- ".set pop \n"
- :
- : "r" (icache_line_size), "r" (icache_sets * icache_assoc),
- "r" (KSEG0), "i" (Index_Invalidate_I));
-}
-
-#ifdef CONFIG_SMP
-extern void sb1___flush_cache_all_ipi(void *ignored);
-asm("sb1___flush_cache_all_ipi = local_sb1___flush_cache_all");
-
-static void sb1___flush_cache_all(void)
-{
- smp_call_function(sb1___flush_cache_all_ipi, 0, 1, 1);
- local_sb1___flush_cache_all();
-}
-#else
-extern void sb1___flush_cache_all(void);
-asm("sb1___flush_cache_all = local_sb1___flush_cache_all");
-#endif
-
-
/*
* When flushing a range in the icache, we have to first writeback
* the dcache for the same range, so new ifetches will see any
@@ -216,33 +150,129 @@
}
#ifdef CONFIG_SMP
-struct flush_icache_range_args {
+
+struct flush_cache_range_args {
unsigned long start;
unsigned long end;
};
static void sb1_flush_icache_range_ipi(void *info)
{
- struct flush_icache_range_args *args = info;
+ struct flush_cache_range_args *args = info;
local_sb1_flush_icache_range(args->start, args->end);
}
-void sb1_flush_icache_range(unsigned long start, unsigned long end)
+void smp_sb1_flush_icache_range(unsigned long start, unsigned long end)
{
- struct flush_icache_range_args args;
+ struct flush_cache_range_args args;
args.start = start;
args.end = end;
smp_call_function(sb1_flush_icache_range_ipi, &args, 1, 1);
local_sb1_flush_icache_range(start, end);
}
-#else
-void sb1_flush_icache_range(unsigned long start, unsigned long end);
-asm("sb1_flush_icache_range = local_sb1_flush_icache_range");
#endif
/*
+ * Writeback and invalidate a range of addresses.
+ */
+static void local_sb1_writeback_inv_dcache_range(unsigned long start, unsigned long end)
+{
+#ifdef CONFIG_SB1_PASS_1_WORKAROUNDS
+ unsigned long flags;
+ local_irq_save(flags);
+#endif
+
+ __asm__ __volatile__ (
+ ".set push \n"
+ ".set noreorder \n"
+ ".set noat \n"
+ ".set mips4 \n"
+ " move $1, %0 \n"
+ "1: \n"
+#ifdef CONFIG_SB1_PASS_1_WORKAROUNDS
+ ".align 3 \n"
+ " lw $0, 0($1) \n" /* Bug 1370, 1368 */
+ " sync \n"
+#endif
+ " cache %3, 0($1) \n" /* Hit-WB{,-inval} this address */
+ " bne $1, %1, 1b \n" /* loop test */
+ " addu $1, $1, %2 \n" /* next line */
+ " sync \n"
+ ".set pop \n"
+ :
+ : "r" (start & ~(dcache_line_size - 1)),
+ "r" ((end - 1) & ~(dcache_line_size - 1)),
+ "r" (dcache_line_size),
+ "i" (Hit_Writeback_Inv_D)
+ );
+#ifdef CONFIG_SB1_PASS_1_WORKAROUNDS
+ local_irq_restore(flags);
+#endif
+}
+
+
+#ifdef CONFIG_SMP
+
+static void sb1_writeback_inv_dcache_range_ipi(void *info)
+{
+ struct flush_cache_range_args *args = info;
+
+ local_sb1_writeback_inv_dcache_range(args->start, args->end);
+}
+
+void smp_sb1_writeback_inv_dcache_range(unsigned long start, unsigned long end)
+{
+ struct flush_cache_range_args args;
+
+ args.start = start;
+ args.end = end;
+ smp_call_function(sb1_writeback_inv_dcache_range_ipi, &args, 1, 1);
+ local_sb1_flush_icache_range(start, end);
+}
+
+#endif
+
+/*
+ * Writeback and invalidate the entire dcache
+ */
+static void local_sb1_writeback_inv_dcache_all(void)
+{
+ /*
+ * Register usage:
+ *
+ * $1 - moving cache index
+ * $2 - set count
+ */
+ __asm__ __volatile__ (
+ ".set push \n"
+ ".set noreorder \n"
+ ".set noat \n"
+ ".set mips4 \n"
+ " move $1, %2 \n" /* Start at index 0 */
+ "1: cache %3, 0($1) \n" /* Invalidate this index */
+ " addiu %1, %1, -1 \n" /* Decrement loop count */
+ " bnez %1, 1b \n" /* loop test */
+ " addu $1, $1, %0 \n" /* Next address */
+ ".set pop \n"
+ :
+ : "r" (dcache_line_size), "r" (dcache_sets * dcache_assoc),
+ "r" (KSEG0), "i" (Index_Writeback_Inv_D));
+}
+
+
+#ifdef CONFIG_SMP
+
+void smp_sb1_writeback_inv_dcache_all(void)
+{
+ smp_call_function((void (*)(void *))local_sb1_writeback_inv_dcache_all, 0, 1, 1);
+ local_sb1_writeback_inv_dcache_all();
+}
+
+#endif
+
+/*
* If there's no context yet, or the page isn't executable, no icache flush
* is needed
*/
@@ -257,7 +287,7 @@
* conservatively flush the entire caches on all processors
* (ouch).
*/
- sb1___flush_cache_all();
+ flush_icache_all();
}
static inline void protected_flush_icache_line(unsigned long addr)
@@ -345,7 +375,7 @@
protected_flush_icache_line(iaddr);
}
-static void sb1_flush_cache_sigtramp(unsigned long addr)
+static void smp_sb1_flush_cache_sigtramp(unsigned long addr)
{
unsigned long tmp;
@@ -359,10 +389,6 @@
smp_call_function(sb1_flush_cache_sigtramp_ipi, (void *) addr, 1, 1);
}
-
-#else
-void sb1_flush_cache_sigtramp(unsigned long addr);
-asm("sb1_flush_cache_sigtramp = local_sb1_flush_cache_sigtramp");
#endif
static void sb1_flush_icache_all(void)
@@ -506,17 +532,26 @@
_copy_page = sb1_copy_page;
_flush_cache_all = sb1_flush_cache_all;
- ___flush_cache_all = sb1___flush_cache_all;
_flush_cache_mm = (void (*)(struct mm_struct *))sb1_nop;
_flush_cache_range = (void *) sb1_nop;
_flush_page_to_ram = sb1_flush_page_to_ram;
_flush_icache_page = sb1_flush_icache_page;
- _flush_icache_range = sb1_flush_icache_range;
+
+#ifdef CONFIG_SMP
+ _writeback_inv_dcache_range = smp_sb1_writeback_inv_dcache_range;
+ _writeback_inv_dcache_all = smp_sb1_writeback_inv_dcache_all;
+ _flush_icache_range = smp_sb1_flush_icache_range;
+ _flush_cache_sigtramp = smp_sb1_flush_cache_sigtramp;
+#else
+ _writeback_inv_dcache_range = local_sb1_writeback_inv_dcache_range;
+ _writeback_inv_dcache_all = local_sb1_writeback_inv_dcache_all;
+ _flush_icache_range = local_sb1_flush_icache_range;
+ _flush_cache_sigtramp = local_sb1_flush_cache_sigtramp;
+#endif
/* None of these are needed for the sb1 */
_flush_cache_page = (void *) sb1_nop;
- _flush_cache_sigtramp = sb1_flush_cache_sigtramp;
_flush_icache_all = sb1_flush_icache_all;
change_cp0_config(CONF_CM_CMASK, CONF_CM_CACHABLE_COW);
Index: arch/mips/mm/init.c
===================================================================
RCS file: /cvs/linux/arch/mips/mm/init.c,v
retrieving revision 1.43
diff -u -r1.43 init.c
--- arch/mips/mm/init.c 2002/03/15 03:14:31 1.43
+++ arch/mips/mm/init.c 2002/05/30 19:13:10
@@ -37,6 +37,7 @@
#include <asm/pgalloc.h>
#include <asm/mmu_context.h>
#include <asm/tlb.h>
+#include <asm/uaccess.h>
mmu_gather_t mmu_gathers[NR_CPUS];
unsigned long highstart_pfn, highend_pfn;
@@ -48,8 +49,21 @@
asmlinkage int sys_cacheflush(void *addr, int bytes, int cache)
{
- /* This should flush more selectivly ... */
- __flush_cache_all();
+ if (cache & ~(ICACHE | DCACHE)) {
+ return -EINVAL;
+ }
+
+ if (verify_area(VERIFY_READ, addr, bytes)) {
+ return -EFAULT;
+ }
+
+ if (cache & DCACHE) {
+ writeback_inv_dcache_range((unsigned long)addr, (unsigned long)addr + bytes);
+ }
+
+ if (cache & ICACHE) {
+ flush_icache_range((unsigned long)addr, ((unsigned long)addr) + bytes);
+ }
return 0;
}
Index: arch/mips/mm/loadmmu.c
===================================================================
RCS file: /cvs/linux/arch/mips/mm/loadmmu.c,v
retrieving revision 1.45
diff -u -r1.45 loadmmu.c
--- arch/mips/mm/loadmmu.c 2001/11/29 04:47:24 1.45
+++ arch/mips/mm/loadmmu.c 2002/05/30 19:13:10
@@ -24,7 +24,6 @@
/* Cache operations. */
void (*_flush_cache_all)(void);
-void (*___flush_cache_all)(void);
void (*_flush_cache_mm)(struct mm_struct *mm);
void (*_flush_cache_range)(struct mm_struct *mm, unsigned long start,
unsigned long end);
@@ -35,6 +34,8 @@
void (*_flush_page_to_ram)(struct page * page);
void (*_flush_icache_all)(void);
+void (*_writeback_inv_dcache_range)(unsigned long start, unsigned long end);
+void (*_writeback_inv_dcache_all) (void);
#ifdef CONFIG_NONCOHERENT_IO
Index: include/asm-mips/pgtable.h
===================================================================
RCS file: /cvs/linux/include/asm-mips/pgtable.h,v
retrieving revision 1.74
diff -u -r1.74 pgtable.h
--- include/asm-mips/pgtable.h 2002/05/28 09:58:58 1.74
+++ include/asm-mips/pgtable.h 2002/05/30 19:13:22
@@ -17,49 +17,70 @@
#include <asm/cachectl.h>
#include <asm/fixmap.h>
-/* Cache flushing:
+/* Generic cache flushing. See Documentation/cachetlb for more specific details
*
- * - flush_cache_all() flushes entire cache
- * - flush_cache_mm(mm) flushes the specified mm context's cache lines
- * - flush_cache_page(mm, vmaddr) flushes a single page
- * - flush_cache_range(mm, start, end) flushes a range of pages
- * - flush_page_to_ram(page) write back kernel page to ram
- * - flush_icache_range(start, end) flush a range of instructions
+ * Note that none of these routines check access permissions. Any needed
+ * checking should be done by the caller.
*
- * - flush_cache_sigtramp() flush signal trampoline
- * - flush_icache_all() flush the entire instruction cache
+ * - flush_cache_all Writeback & inval all virtually mapped cached data
+ * & ensure all writes are visible to the system.
+ * - flush_cache_mm Same as above, but only for a specific mm context
+ * - flush_cache_page Same as above, but for a single page
+ * - flush_cache_range Same as above, for a range of addresses
+ * - flush_page_to_ram Clear all virtual mappings of this physical page
+ * - flush_icache_range An istream modification may have occurred; ensure
+ * that all data stores before this point are visible
+ * for new instruction fetches for the given range.
+ * - flush_icache_page Same as above, but for a specific page
*/
-extern void (*_flush_cache_all)(void);
-extern void (*___flush_cache_all)(void);
-extern void (*_flush_cache_mm)(struct mm_struct *mm);
-extern void (*_flush_cache_range)(struct mm_struct *mm, unsigned long start,
- unsigned long end);
-extern void (*_flush_cache_page)(struct vm_area_struct *vma, unsigned long page);
-extern void (*_flush_page_to_ram)(struct page * page);
+
+/*
+ * Mips-specific cache flushing.
+ * - flush_icache_all Same as flush_icache_range, for all memory
+ * - writeback_inv_dcache_all Force writeback and invalidation of the first
+ * level dcache.
+ * - writeback_inv_dcache_range Same as above, but for a specific virtual range
+ * - flush_cache_sigtramp Flush signal trampoline
+ */
+
+extern void (*_flush_cache_all) (void);
+extern void (*_flush_cache_mm) (struct mm_struct *mm);
+extern void (*_flush_cache_page) (struct vm_area_struct *vma,
+ unsigned long page);
+extern void (*_flush_cache_range) (struct mm_struct *mm, unsigned long start,
+ unsigned long end);
+extern void (*_flush_page_to_ram) (struct page * page);
extern void (*_flush_icache_range)(unsigned long start, unsigned long end);
-extern void (*_flush_icache_page)(struct vm_area_struct *vma,
- struct page *page);
-extern void (*_flush_cache_sigtramp)(unsigned long addr);
-extern void (*_flush_icache_all)(void);
+extern void (*_flush_icache_page) (struct vm_area_struct *vma,
+ struct page *page);
+
+extern void (*_flush_icache_all) (void);
+extern void (*_writeback_inv_dcache_all) (void);
+extern void (*_writeback_inv_dcache_range)(unsigned long start,
+ unsigned long end);
+extern void (*_flush_cache_sigtramp) (unsigned long addr);
+
#define flush_dcache_page(page) do { } while (0)
#define flush_cache_all() _flush_cache_all()
-#define __flush_cache_all() ___flush_cache_all()
#define flush_cache_mm(mm) _flush_cache_mm(mm)
-#define flush_cache_range(mm,start,end) _flush_cache_range(mm,start,end)
#define flush_cache_page(vma,page) _flush_cache_page(vma, page)
+#define flush_cache_range(mm,start,end) _flush_cache_range(mm,start,end)
#define flush_page_to_ram(page) _flush_page_to_ram(page)
-
#define flush_icache_range(start, end) _flush_icache_range(start,end)
#define flush_icache_page(vma, page) _flush_icache_page(vma, page)
+
+
-#define flush_cache_sigtramp(addr) _flush_cache_sigtramp(addr)
#ifdef CONFIG_VTAG_ICACHE
-#define flush_icache_all() _flush_icache_all()
+#define flush_icache_all() _flush_icache_all()
#else
-#define flush_icache_all() do { } while(0)
+#define flush_icache_all() do { } while(0)
#endif
+#define writeback_inv_dcache_all() _writeback_inv_dcache_all()
+#define writeback_inv_dcache_range(start, end) _writeback_inv_dcache_range(start, end)
+#define flush_cache_sigtramp(addr) _flush_cache_sigtramp(addr)
/*
* - add_wired_entry() add a fixed TLB entry, and move wired register
