The attached patch implements three optimizations:
1) Loops in flush_user_dcache_range_asm, flush_kernel_dcache_range_asm,
purge_kernel_dcache_range_asm,
flush_user_icache_range_asm, and flush_kernel_icache_range_asm are
unrolled to reduce branch overhead.
2) The static branch prediction for cmpb instructions in pacache.S has
been reviewed and the operand
order adjusted where necessary.
3) For flush routines in cache.c, we purge rather flush when we have no
context. The pdc instruction
at level 0 is not required to write back dirty lines to memory. This
provides a performance improvement
over the fdc instruction if the feature is implemented.
Unfortunately, this patch didn't provide any significant improvement in
gcc-8 build and check times on c8000.
At best, it's ten minutes in a total time of 20:33 hours.
I've posted the patch as it might provide more benefit on other machines.
Signed-off-by: John David Anglin <dave.anglin@xxxxxxxx>
diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c
index e3b45546d589..ab5bf0e8aafc 100644
--- a/arch/parisc/kernel/cache.c
+++ b/arch/parisc/kernel/cache.c
@@ -36,6 +36,7 @@ EXPORT_SYMBOL(dcache_stride);
void flush_dcache_page_asm(unsigned long phys_addr, unsigned long vaddr);
EXPORT_SYMBOL(flush_dcache_page_asm);
+void purge_dcache_page_asm(unsigned long phys_addr, unsigned long vaddr);
void flush_icache_page_asm(unsigned long phys_addr, unsigned long vaddr);
@@ -302,6 +303,17 @@ __flush_cache_page(struct vm_area_struct *vma, unsigned long vmaddr,
preempt_enable();
}
+static inline void
+__purge_cache_page(struct vm_area_struct *vma, unsigned long vmaddr,
+ unsigned long physaddr)
+{
+ preempt_disable();
+ purge_dcache_page_asm(physaddr, vmaddr);
+ if (vma->vm_flags & VM_EXEC)
+ flush_icache_page_asm(physaddr, vmaddr);
+ preempt_enable();
+}
+
void flush_dcache_page(struct page *page)
{
struct address_space *mapping = page_mapping(page);
@@ -572,9 +584,12 @@ void flush_cache_mm(struct mm_struct *mm)
pfn = pte_pfn(*ptep);
if (!pfn_valid(pfn))
continue;
- if (unlikely(mm->context))
+ if (unlikely(mm->context)) {
flush_tlb_page(vma, addr);
- __flush_cache_page(vma, addr, PFN_PHYS(pfn));
+ __flush_cache_page(vma, addr, PFN_PHYS(pfn));
+ } else {
+ __purge_cache_page(vma, addr, PFN_PHYS(pfn));
+ }
}
}
}
@@ -609,9 +624,12 @@ void flush_cache_range(struct vm_area_struct *vma,
continue;
pfn = pte_pfn(*ptep);
if (pfn_valid(pfn)) {
- if (unlikely(vma->vm_mm->context))
+ if (unlikely(vma->vm_mm->context)) {
flush_tlb_page(vma, addr);
- __flush_cache_page(vma, addr, PFN_PHYS(pfn));
+ __flush_cache_page(vma, addr, PFN_PHYS(pfn));
+ } else {
+ __purge_cache_page(vma, addr, PFN_PHYS(pfn));
+ }
}
}
}
@@ -620,9 +638,12 @@ void
flush_cache_page(struct vm_area_struct *vma, unsigned long vmaddr, unsigned long pfn)
{
if (pfn_valid(pfn)) {
- if (likely(vma->vm_mm->context))
+ if (likely(vma->vm_mm->context)) {
flush_tlb_page(vma, vmaddr);
- __flush_cache_page(vma, vmaddr, PFN_PHYS(pfn));
+ __flush_cache_page(vma, vmaddr, PFN_PHYS(pfn));
+ } else {
+ __purge_cache_page(vma, vmaddr, PFN_PHYS(pfn));
+ }
}
}
diff --git a/arch/parisc/kernel/pacache.S b/arch/parisc/kernel/pacache.S
index 67b0f7532e83..1cff7f3c2af2 100644
--- a/arch/parisc/kernel/pacache.S
+++ b/arch/parisc/kernel/pacache.S
@@ -881,7 +881,6 @@ ENTRY_CFI(flush_dcache_page_asm)
add %r28, %r25, %r25
sub %r25, r31, %r25
-
1: fdc,m r31(%r28)
fdc,m r31(%r28)
fdc,m r31(%r28)
@@ -897,7 +896,7 @@ ENTRY_CFI(flush_dcache_page_asm)
fdc,m r31(%r28)
fdc,m r31(%r28)
fdc,m r31(%r28)
- cmpb,COND(<<) %r28, %r25,1b
+ cmpb,COND(>>) %r25, %r28, 1b /* predict taken */
fdc,m r31(%r28)
sync
@@ -908,6 +907,72 @@ ENTRY_CFI(flush_dcache_page_asm)
.procend
ENDPROC_CFI(flush_dcache_page_asm)
+ENTRY_CFI(purge_dcache_page_asm)
+ .proc
+ .callinfo NO_CALLS
+ .entry
+
+ ldil L%(TMPALIAS_MAP_START), %r28
+#ifdef CONFIG_64BIT
+#if (TMPALIAS_MAP_START >= 0x80000000)
+ depdi 0, 31,32, %r28 /* clear any sign extension */
+#endif
+ convert_phys_for_tlb_insert20 %r26 /* convert phys addr to tlb insert format */
+ depd %r25, 63,22, %r28 /* Form aliased virtual address 'to' */
+ depdi 0, 63,PAGE_SHIFT, %r28 /* Clear any offset bits */
+#else
+ extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
+ depw %r25, 31,22, %r28 /* Form aliased virtual address 'to' */
+ depwi 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */
+#endif
+
+ /* Purge any old translation */
+
+#ifdef CONFIG_PA20
+ pdtlb,l %r0(%r28)
+#else
+ tlb_lock %r20,%r21,%r22
+ pdtlb %r0(%r28)
+ tlb_unlock %r20,%r21,%r22
+#endif
+
+ ldil L%dcache_stride, %r1
+ ldw R%dcache_stride(%r1), r31
+
+#ifdef CONFIG_64BIT
+ depdi,z 1, 63-PAGE_SHIFT,1, %r25
+#else
+ depwi,z 1, 31-PAGE_SHIFT,1, %r25
+#endif
+ add %r28, %r25, %r25
+ sub %r25, r31, %r25
+
+1: pdc,m r31(%r28)
+ pdc,m r31(%r28)
+ pdc,m r31(%r28)
+ pdc,m r31(%r28)
+ pdc,m r31(%r28)
+ pdc,m r31(%r28)
+ pdc,m r31(%r28)
+ pdc,m r31(%r28)
+ pdc,m r31(%r28)
+ pdc,m r31(%r28)
+ pdc,m r31(%r28)
+ pdc,m r31(%r28)
+ pdc,m r31(%r28)
+ pdc,m r31(%r28)
+ pdc,m r31(%r28)
+ cmpb,COND(>>) %r25, %r28, 1b /* predict taken */
+ pdc,m r31(%r28)
+
+ sync
+ bv %r0(%r2)
+ nop
+ .exit
+
+ .procend
+ENDPROC_CFI(purge_dcache_page_asm)
+
ENTRY_CFI(flush_icache_page_asm)
.proc
.callinfo NO_CALLS
@@ -953,7 +1018,6 @@ ENTRY_CFI(flush_icache_page_asm)
add %r28, %r25, %r25
sub %r25, %r31, %r25
-
/* fic only has the type 26 form on PA1.1, requiring an
* explicit space specification, so use %sr4 */
1: fic,m %r31(%sr4,%r28)
@@ -971,7 +1035,7 @@ ENTRY_CFI(flush_icache_page_asm)
fic,m %r31(%sr4,%r28)
fic,m %r31(%sr4,%r28)
fic,m %r31(%sr4,%r28)
- cmpb,COND(<<) %r28, %r25,1b
+ cmpb,COND(>>) %r25, %r28, 1b /* predict taken */
fic,m %r31(%sr4,%r28)
sync
@@ -998,7 +1062,6 @@ ENTRY_CFI(flush_kernel_dcache_page_asm)
add %r26, %r25, %r25
sub %r25, %r23, %r25
-
1: fdc,m %r23(%r26)
fdc,m %r23(%r26)
fdc,m %r23(%r26)
@@ -1014,7 +1077,7 @@ ENTRY_CFI(flush_kernel_dcache_page_asm)
fdc,m %r23(%r26)
fdc,m %r23(%r26)
fdc,m %r23(%r26)
- cmpb,COND(<<) %r26, %r25,1b
+ cmpb,COND(>>) %r25, %r26, 1b /* predict taken */
fdc,m %r23(%r26)
sync
@@ -1056,7 +1119,7 @@ ENTRY_CFI(purge_kernel_dcache_page_asm)
pdc,m %r23(%r26)
pdc,m %r23(%r26)
pdc,m %r23(%r26)
- cmpb,COND(<<) %r26, %r25, 1b
+ cmpb,COND(>>) %r25, %r26, 1b /* predict taken */
pdc,m %r23(%r26)
sync
@@ -1077,7 +1140,33 @@ ENTRY_CFI(flush_user_dcache_range_asm)
ldo -1(%r23), %r21
ANDCM %r26, %r21, %r26
-1: cmpb,COND(<<),n %r26, %r25, 1b
+#ifdef CONFIG_64BIT
+ depd,z %r23, 59, 60, %r21
+#else
+ depw,z %r23, 27, 28, %r21
+#endif
+ add %r26, %r21, %r22
+ cmpb,COND(>>),n %r22, %r25, 2f /* predict not taken */
+1: add %r22, %r21, %r22
+ fdc,m %r23(%sr3, %r26)
+ fdc,m %r23(%sr3, %r26)
+ fdc,m %r23(%sr3, %r26)
+ fdc,m %r23(%sr3, %r26)
+ fdc,m %r23(%sr3, %r26)
+ fdc,m %r23(%sr3, %r26)
+ fdc,m %r23(%sr3, %r26)
+ fdc,m %r23(%sr3, %r26)
+ fdc,m %r23(%sr3, %r26)
+ fdc,m %r23(%sr3, %r26)
+ fdc,m %r23(%sr3, %r26)
+ fdc,m %r23(%sr3, %r26)
+ fdc,m %r23(%sr3, %r26)
+ fdc,m %r23(%sr3, %r26)
+ fdc,m %r23(%sr3, %r26)
+ cmpb,COND(<<=) %r22, %r25, 1b /* predict taken */
+ fdc,m %r23(%sr3, %r26)
+
+2: cmpb,COND(>>),n %r25, %r26, 2b
fdc,m %r23(%sr3, %r26)
sync
@@ -1098,7 +1187,33 @@ ENTRY_CFI(flush_kernel_dcache_range_asm)
ldo -1(%r23), %r21
ANDCM %r26, %r21, %r26
-1: cmpb,COND(<<),n %r26, %r25,1b
+#ifdef CONFIG_64BIT
+ depd,z %r23, 59, 60, %r21
+#else
+ depw,z %r23, 27, 28, %r21
+#endif
+ add %r26, %r21, %r22
+ cmpb,COND(>>),n %r22, %r25, 2f /* predict not taken */
+1: add %r22, %r21, %r22
+ fdc,m %r23(%r26)
+ fdc,m %r23(%r26)
+ fdc,m %r23(%r26)
+ fdc,m %r23(%r26)
+ fdc,m %r23(%r26)
+ fdc,m %r23(%r26)
+ fdc,m %r23(%r26)
+ fdc,m %r23(%r26)
+ fdc,m %r23(%r26)
+ fdc,m %r23(%r26)
+ fdc,m %r23(%r26)
+ fdc,m %r23(%r26)
+ fdc,m %r23(%r26)
+ fdc,m %r23(%r26)
+ fdc,m %r23(%r26)
+ cmpb,COND(<<=) %r22, %r25, 1b /* predict taken */
+ fdc,m %r23(%r26)
+
+2: cmpb,COND(>>),n %r25, %r26, 2b /* predict taken */
fdc,m %r23(%r26)
sync
@@ -1120,7 +1235,33 @@ ENTRY_CFI(purge_kernel_dcache_range_asm)
ldo -1(%r23), %r21
ANDCM %r26, %r21, %r26
-1: cmpb,COND(<<),n %r26, %r25,1b
+#ifdef CONFIG_64BIT
+ depd,z %r23, 59, 60, %r21
+#else
+ depw,z %r23, 27, 28, %r21
+#endif
+ add %r26, %r21, %r22
+ cmpb,COND(>>),n %r22, %r25, 2f /* predict not taken */
+1: add %r22, %r21, %r22
+ pdc,m %r23(%r26)
+ pdc,m %r23(%r26)
+ pdc,m %r23(%r26)
+ pdc,m %r23(%r26)
+ pdc,m %r23(%r26)
+ pdc,m %r23(%r26)
+ pdc,m %r23(%r26)
+ pdc,m %r23(%r26)
+ pdc,m %r23(%r26)
+ pdc,m %r23(%r26)
+ pdc,m %r23(%r26)
+ pdc,m %r23(%r26)
+ pdc,m %r23(%r26)
+ pdc,m %r23(%r26)
+ pdc,m %r23(%r26)
+ cmpb,COND(<<=) %r22, %r25, 1b /* predict taken */
+ pdc,m %r23(%r26)
+
+2: cmpb,COND(>>),n %r25, %r26, 2b /* predict taken */
pdc,m %r23(%r26)
sync
@@ -1142,7 +1283,33 @@ ENTRY_CFI(flush_user_icache_range_asm)
ldo -1(%r23), %r21
ANDCM %r26, %r21, %r26
-1: cmpb,COND(<<),n %r26, %r25,1b
+#ifdef CONFIG_64BIT
+ depd,z %r23, 59, 60, %r21
+#else
+ depw,z %r23, 27, 28, %r21
+#endif
+ add %r26, %r21, %r22
+ cmpb,COND(>>),n %r22, %r25, 2f /* predict not taken */
+1: add %r22, %r21, %r22
+ fic,m %r23(%sr3, %r26)
+ fic,m %r23(%sr3, %r26)
+ fic,m %r23(%sr3, %r26)
+ fic,m %r23(%sr3, %r26)
+ fic,m %r23(%sr3, %r26)
+ fic,m %r23(%sr3, %r26)
+ fic,m %r23(%sr3, %r26)
+ fic,m %r23(%sr3, %r26)
+ fic,m %r23(%sr3, %r26)
+ fic,m %r23(%sr3, %r26)
+ fic,m %r23(%sr3, %r26)
+ fic,m %r23(%sr3, %r26)
+ fic,m %r23(%sr3, %r26)
+ fic,m %r23(%sr3, %r26)
+ fic,m %r23(%sr3, %r26)
+ cmpb,COND(<<=) %r22, %r25, 1b /* predict taken */
+ fic,m %r23(%sr3, %r26)
+
+2: cmpb,COND(>>),n %r25, %r26, 2b
fic,m %r23(%sr3, %r26)
sync
@@ -1185,7 +1352,7 @@ ENTRY_CFI(flush_kernel_icache_page)
fic,m %r23(%sr4, %r26)
fic,m %r23(%sr4, %r26)
fic,m %r23(%sr4, %r26)
- cmpb,COND(<<) %r26, %r25, 1b
+ cmpb,COND(>>) %r25, %r26, 1b /* predict taken */
fic,m %r23(%sr4, %r26)
sync
@@ -1206,7 +1373,33 @@ ENTRY_CFI(flush_kernel_icache_range_asm)
ldo -1(%r23), %r21
ANDCM %r26, %r21, %r26
-1: cmpb,COND(<<),n %r26, %r25, 1b
+#ifdef CONFIG_64BIT
+ depd,z %r23, 59, 60, %r21
+#else
+ depw,z %r23, 27, 28, %r21
+#endif
+ add %r26, %r21, %r22
+ cmpb,COND(>>),n %r22, %r25, 2f /* predict not taken */
+1: add %r22, %r21, %r22
+ fic,m %r23(%sr4, %r26)
+ fic,m %r23(%sr4, %r26)
+ fic,m %r23(%sr4, %r26)
+ fic,m %r23(%sr4, %r26)
+ fic,m %r23(%sr4, %r26)
+ fic,m %r23(%sr4, %r26)
+ fic,m %r23(%sr4, %r26)
+ fic,m %r23(%sr4, %r26)
+ fic,m %r23(%sr4, %r26)
+ fic,m %r23(%sr4, %r26)
+ fic,m %r23(%sr4, %r26)
+ fic,m %r23(%sr4, %r26)
+ fic,m %r23(%sr4, %r26)
+ fic,m %r23(%sr4, %r26)
+ fic,m %r23(%sr4, %r26)
+ cmpb,COND(<<=) %r22, %r25, 1b /* predict taken */
+ fic,m %r23(%sr4, %r26)
+
+2: cmpb,COND(>>),n %r25, %r26, 2b /* predict taken */
fic,m %r23(%sr4, %r26)
sync
