On the Loongson-2G/2H/3A/3B there is a hardware flaw that ll/sc and
lld/scd is very weak ordering. We should add sync instructions before
each ll/lld and after the last sc/scd to workaround. Otherwise, this
flaw will cause deadlock occationally (e.g. when doing heavy load test
with LTP).
This patch is not a minimal change (it is very difficult to make a
minimal change), but it is a safest change.
Why disable fix-loongson3-llsc in compiler?
Because compiler fix will cause problems in kernel's .fixup section.
Signed-off-by: Huacai Chen <chenhc@xxxxxxxxxx>
---
arch/mips/include/asm/atomic.h | 36 ++++++++++++++++++++++++++++--------
arch/mips/include/asm/barrier.h | 6 ++++++
arch/mips/include/asm/bitops.h | 15 +++++++++++++++
arch/mips/include/asm/cmpxchg.h | 9 +++++++--
arch/mips/include/asm/edac.h | 5 ++++-
arch/mips/include/asm/futex.h | 18 ++++++++++++------
arch/mips/include/asm/local.h | 10 ++++++++--
arch/mips/include/asm/pgtable.h | 5 ++++-
arch/mips/kernel/syscall.c | 2 ++
arch/mips/loongson64/Platform | 3 +++
arch/mips/mm/tlbex.c | 11 +++++++++++
11 files changed, 100 insertions(+), 20 deletions(-)
diff --git a/arch/mips/include/asm/atomic.h b/arch/mips/include/asm/atomic.h
index e8fbfd4..9fbe85f 100644
--- a/arch/mips/include/asm/atomic.h
+++ b/arch/mips/include/asm/atomic.h
@@ -61,13 +61,16 @@ static __inline__ void atomic_##op(int i, atomic_t * v) \
__asm__ __volatile__( \
" .set push \n" \
" .set "MIPS_ISA_LEVEL" \n" \
- "1: ll %0, %1 # atomic_" #op " \n" \
+ "1: # atomic_" #op " \n" \
+ __WAR_LLSC_MB \
+ " ll %0, %1 \n" \
" " #asm_op " %0, %2 \n" \
" sc %0, %1 \n" \
"\t" __scbeqz " %0, 1b \n" \
" .set pop \n" \
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter) \
: "Ir" (i)); \
+ __asm__ __volatile__(__WAR_LLSC_MB : : :"memory"); \
} else { \
unsigned long flags; \
\
@@ -88,7 +91,9 @@ static __inline__ int atomic_##op##_return_relaxed(int i, atomic_t * v) \
__asm__ __volatile__( \
" .set push \n" \
" .set "MIPS_ISA_LEVEL" \n" \
- "1: ll %1, %2 # atomic_" #op "_return \n" \
+ "1: # atomic_" #op "_return \n" \
+ __WAR_LLSC_MB \
+ " ll %1, %2 \n" \
" " #asm_op " %0, %1, %3 \n" \
" sc %0, %2 \n" \
"\t" __scbeqz " %0, 1b \n" \
@@ -121,7 +126,9 @@ static __inline__ int atomic_fetch_##op##_relaxed(int i, atomic_t * v) \
__asm__ __volatile__( \
" .set push \n" \
" .set "MIPS_ISA_LEVEL" \n" \
- "1: ll %1, %2 # atomic_fetch_" #op " \n" \
+ "1: # atomic_fetch_" #op " \n" \
+ __WAR_LLSC_MB \
+ " ll %1, %2 \n" \
" " #asm_op " %0, %1, %3 \n" \
" sc %0, %2 \n" \
"\t" __scbeqz " %0, 1b \n" \
@@ -130,6 +137,7 @@ static __inline__ int atomic_fetch_##op##_relaxed(int i, atomic_t * v) \
: "=&r" (result), "=&r" (temp), \
"+" GCC_OFF_SMALL_ASM() (v->counter) \
: "Ir" (i)); \
+ __asm__ __volatile__(__WAR_LLSC_MB : : :"memory"); \
} else { \
unsigned long flags; \
\
@@ -193,7 +201,9 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
__asm__ __volatile__(
" .set push \n"
" .set "MIPS_ISA_LEVEL" \n"
- "1: ll %1, %2 # atomic_sub_if_positive\n"
+ "1: # atomic_sub_if_positive\n"
+ __WAR_LLSC_MB
+ " ll %1, %2 \n"
" .set pop \n"
" subu %0, %1, %3 \n"
" move %1, %0 \n"
@@ -259,13 +269,16 @@ static __inline__ void atomic64_##op(long i, atomic64_t * v) \
__asm__ __volatile__( \
" .set push \n" \
" .set "MIPS_ISA_LEVEL" \n" \
- "1: lld %0, %1 # atomic64_" #op " \n" \
+ "1: # atomic64_" #op " \n" \
+ __WAR_LLSC_MB \
+ " lld %0, %1 \n" \
" " #asm_op " %0, %2 \n" \
" scd %0, %1 \n" \
"\t" __scbeqz " %0, 1b \n" \
" .set pop \n" \
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter) \
: "Ir" (i)); \
+ __asm__ __volatile__(__WAR_LLSC_MB : : :"memory"); \
} else { \
unsigned long flags; \
\
@@ -286,7 +299,9 @@ static __inline__ long atomic64_##op##_return_relaxed(long i, atomic64_t * v) \
__asm__ __volatile__( \
" .set push \n" \
" .set "MIPS_ISA_LEVEL" \n" \
- "1: lld %1, %2 # atomic64_" #op "_return\n" \
+ "1: # atomic64_" #op "_return\n" \
+ __WAR_LLSC_MB \
+ " lld %1, %2 \n" \
" " #asm_op " %0, %1, %3 \n" \
" scd %0, %2 \n" \
"\t" __scbeqz " %0, 1b \n" \
@@ -319,7 +334,9 @@ static __inline__ long atomic64_fetch_##op##_relaxed(long i, atomic64_t * v) \
__asm__ __volatile__( \
" .set push \n" \
" .set "MIPS_ISA_LEVEL" \n" \
- "1: lld %1, %2 # atomic64_fetch_" #op "\n" \
+ "1: # atomic64_fetch_" #op "\n" \
+ __WAR_LLSC_MB \
+ " lld %1, %2 \n" \
" " #asm_op " %0, %1, %3 \n" \
" scd %0, %2 \n" \
"\t" __scbeqz " %0, 1b \n" \
@@ -328,6 +345,7 @@ static __inline__ long atomic64_fetch_##op##_relaxed(long i, atomic64_t * v) \
: "=&r" (result), "=&r" (temp), \
"+" GCC_OFF_SMALL_ASM() (v->counter) \
: "Ir" (i)); \
+ __asm__ __volatile__(__WAR_LLSC_MB : : :"memory"); \
} else { \
unsigned long flags; \
\
@@ -392,7 +410,9 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
__asm__ __volatile__(
" .set push \n"
" .set "MIPS_ISA_LEVEL" \n"
- "1: lld %1, %2 # atomic64_sub_if_positive\n"
+ "1: # atomic64_sub_if_positive\n"
+ __WAR_LLSC_MB
+ " lld %1, %2 \n"
" dsubu %0, %1, %3 \n"
" move %1, %0 \n"
" bltz %0, 1f \n"
diff --git a/arch/mips/include/asm/barrier.h b/arch/mips/include/asm/barrier.h
index a5eb1bb..d892094 100644
--- a/arch/mips/include/asm/barrier.h
+++ b/arch/mips/include/asm/barrier.h
@@ -203,6 +203,12 @@
#define __WEAK_LLSC_MB " \n"
#endif
+#if defined(CONFIG_LOONGSON3) && defined(CONFIG_SMP) /* Loongson-3's LLSC workaround */
+#define __WAR_LLSC_MB " sync \n"
+#else
+#define __WAR_LLSC_MB " \n"
+#endif
+
#define smp_llsc_mb() __asm__ __volatile__(__WEAK_LLSC_MB : : :"memory")
#ifdef CONFIG_CPU_CAVIUM_OCTEON
diff --git a/arch/mips/include/asm/bitops.h b/arch/mips/include/asm/bitops.h
index f2a840f..4943f52 100644
--- a/arch/mips/include/asm/bitops.h
+++ b/arch/mips/include/asm/bitops.h
@@ -71,18 +71,21 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
} else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
do {
__asm__ __volatile__(
+ __WAR_LLSC_MB
" " __LL "%0, %1 # set_bit \n"
" " __INS "%0, %3, %2, 1 \n"
" " __SC "%0, %1 \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
: "ir" (bit), "r" (~0));
} while (unlikely(!temp));
+ __asm__ __volatile__(__WAR_LLSC_MB : : :"memory");
#endif /* CONFIG_CPU_MIPSR2 || CONFIG_CPU_MIPSR6 */
} else if (kernel_uses_llsc) {
do {
__asm__ __volatile__(
" .set push \n"
" .set "MIPS_ISA_ARCH_LEVEL" \n"
+ __WAR_LLSC_MB
" " __LL "%0, %1 # set_bit \n"
" or %0, %2 \n"
" " __SC "%0, %1 \n"
@@ -90,6 +93,7 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
: "ir" (1UL << bit));
} while (unlikely(!temp));
+ __asm__ __volatile__(__WAR_LLSC_MB : : :"memory");
} else
__mips_set_bit(nr, addr);
}
@@ -125,18 +129,21 @@ static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
} else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
do {
__asm__ __volatile__(
+ __WAR_LLSC_MB
" " __LL "%0, %1 # clear_bit \n"
" " __INS "%0, $0, %2, 1 \n"
" " __SC "%0, %1 \n"
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
: "ir" (bit));
} while (unlikely(!temp));
+ __asm__ __volatile__(__WAR_LLSC_MB : : :"memory");
#endif /* CONFIG_CPU_MIPSR2 || CONFIG_CPU_MIPSR6 */
} else if (kernel_uses_llsc) {
do {
__asm__ __volatile__(
" .set push \n"
" .set "MIPS_ISA_ARCH_LEVEL" \n"
+ __WAR_LLSC_MB
" " __LL "%0, %1 # clear_bit \n"
" and %0, %2 \n"
" " __SC "%0, %1 \n"
@@ -144,6 +151,7 @@ static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
: "ir" (~(1UL << bit)));
} while (unlikely(!temp));
+ __asm__ __volatile__(__WAR_LLSC_MB : : :"memory");
} else
__mips_clear_bit(nr, addr);
}
@@ -197,6 +205,7 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
__asm__ __volatile__(
" .set push \n"
" .set "MIPS_ISA_ARCH_LEVEL" \n"
+ __WAR_LLSC_MB
" " __LL "%0, %1 # change_bit \n"
" xor %0, %2 \n"
" " __SC "%0, %1 \n"
@@ -204,6 +213,7 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
: "ir" (1UL << bit));
} while (unlikely(!temp));
+ __asm__ __volatile__(__WAR_LLSC_MB : : :"memory");
} else
__mips_change_bit(nr, addr);
}
@@ -248,6 +258,7 @@ static inline int test_and_set_bit(unsigned long nr,
__asm__ __volatile__(
" .set push \n"
" .set "MIPS_ISA_ARCH_LEVEL" \n"
+ __WAR_LLSC_MB
" " __LL "%0, %1 # test_and_set_bit \n"
" or %2, %0, %3 \n"
" " __SC "%2, %1 \n"
@@ -304,6 +315,7 @@ static inline int test_and_set_bit_lock(unsigned long nr,
__asm__ __volatile__(
" .set push \n"
" .set "MIPS_ISA_ARCH_LEVEL" \n"
+ __WAR_LLSC_MB
" " __LL "%0, %1 # test_and_set_bit \n"
" or %2, %0, %3 \n"
" " __SC "%2, %1 \n"
@@ -361,6 +373,7 @@ static inline int test_and_clear_bit(unsigned long nr,
do {
__asm__ __volatile__(
+ __WAR_LLSC_MB
" " __LL "%0, %1 # test_and_clear_bit \n"
" " __EXT "%2, %0, %3, 1 \n"
" " __INS "%0, $0, %3, 1 \n"
@@ -378,6 +391,7 @@ static inline int test_and_clear_bit(unsigned long nr,
__asm__ __volatile__(
" .set push \n"
" .set "MIPS_ISA_ARCH_LEVEL" \n"
+ __WAR_LLSC_MB
" " __LL "%0, %1 # test_and_clear_bit \n"
" or %2, %0, %3 \n"
" xor %2, %3 \n"
@@ -437,6 +451,7 @@ static inline int test_and_change_bit(unsigned long nr,
__asm__ __volatile__(
" .set push \n"
" .set "MIPS_ISA_ARCH_LEVEL" \n"
+ __WAR_LLSC_MB
" " __LL "%0, %1 # test_and_change_bit \n"
" xor %2, %0, %3 \n"
" " __SC "\t%2, %1 \n"
diff --git a/arch/mips/include/asm/cmpxchg.h b/arch/mips/include/asm/cmpxchg.h
index 638de0c..8bf2f23 100644
--- a/arch/mips/include/asm/cmpxchg.h
+++ b/arch/mips/include/asm/cmpxchg.h
@@ -49,7 +49,9 @@ extern unsigned long __xchg_called_with_bad_pointer(void)
" .set noat \n" \
" .set push \n" \
" .set " MIPS_ISA_ARCH_LEVEL " \n" \
- "1: " ld " %0, %2 # __xchg_asm \n" \
+ "1: # __xchg_asm \n" \
+ __WAR_LLSC_MB \
+ " " ld " %0, %2 \n" \
" .set pop \n" \
" move $1, %z3 \n" \
" .set " MIPS_ISA_ARCH_LEVEL " \n" \
@@ -120,7 +122,9 @@ static inline unsigned long __xchg(volatile void *ptr, unsigned long x,
" .set noat \n" \
" .set push \n" \
" .set "MIPS_ISA_ARCH_LEVEL" \n" \
- "1: " ld " %0, %2 # __cmpxchg_asm \n" \
+ "1: # __cmpxchg_asm \n" \
+ __WAR_LLSC_MB \
+ " " ld " %0, %2 \n" \
" bne %0, %z3, 2f \n" \
" .set pop \n" \
" move $1, %z4 \n" \
@@ -129,6 +133,7 @@ static inline unsigned long __xchg(volatile void *ptr, unsigned long x,
"\t" __scbeqz " $1, 1b \n" \
" .set pop \n" \
"2: \n" \
+ __WAR_LLSC_MB \
: "=&r" (__ret), "=" GCC_OFF_SMALL_ASM() (*m) \
: GCC_OFF_SMALL_ASM() (*m), "Jr" (old), "Jr" (new) \
: "memory"); \
diff --git a/arch/mips/include/asm/edac.h b/arch/mips/include/asm/edac.h
index c5d1477..59b776d 100644
--- a/arch/mips/include/asm/edac.h
+++ b/arch/mips/include/asm/edac.h
@@ -23,13 +23,16 @@ static inline void edac_atomic_scrub(void *va, u32 size)
__asm__ __volatile__ (
" .set push \n"
" .set mips2 \n"
- "1: ll %0, %1 # edac_atomic_scrub \n"
+ "1: # edac_atomic_scrub \n"
+ __WAR_LLSC_MB
+ " ll %0, %1 \n"
" addu %0, $0 \n"
" sc %0, %1 \n"
" beqz %0, 1b \n"
" .set pop \n"
: "=&r" (temp), "=" GCC_OFF_SMALL_ASM() (*virt_addr)
: GCC_OFF_SMALL_ASM() (*virt_addr));
+ __asm__ __volatile__(__WAR_LLSC_MB : : :"memory");
virt_addr++;
}
diff --git a/arch/mips/include/asm/futex.h b/arch/mips/include/asm/futex.h
index 8eff134..9887c34 100644
--- a/arch/mips/include/asm/futex.h
+++ b/arch/mips/include/asm/futex.h
@@ -55,7 +55,9 @@
" .set noat \n" \
" .set push \n" \
" .set "MIPS_ISA_ARCH_LEVEL" \n" \
- "1: "user_ll("%1", "%4")" # __futex_atomic_op\n" \
+ "1: # __futex_atomic_op\n" \
+ __WAR_LLSC_MB \
+ " "user_ll("%1", "%4")" \n" \
" .set pop \n" \
" " insn " \n" \
" .set "MIPS_ISA_ARCH_LEVEL" \n" \
@@ -70,8 +72,9 @@
" j 3b \n" \
" .previous \n" \
" .section __ex_table,\"a\" \n" \
- " "__UA_ADDR "\t1b, 4b \n" \
- " "__UA_ADDR "\t2b, 4b \n" \
+ " "__UA_ADDR "\t(1b + 0), 4b \n" \
+ " "__UA_ADDR "\t(1b + 4), 4b \n" \
+ " "__UA_ADDR "\t(2b + 0), 4b \n" \
" .previous \n" \
: "=r" (ret), "=&r" (oldval), \
"=" GCC_OFF_SMALL_ASM() (*uaddr) \
@@ -169,7 +172,9 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
" .set noat \n"
" .set push \n"
" .set "MIPS_ISA_ARCH_LEVEL" \n"
- "1: "user_ll("%1", "%3")" \n"
+ "1: \n"
+ __WAR_LLSC_MB
+ " "user_ll("%1", "%3")" \n"
" bne %1, %z4, 3f \n"
" .set pop \n"
" move $1, %z5 \n"
@@ -185,8 +190,9 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
" j 3b \n"
" .previous \n"
" .section __ex_table,\"a\" \n"
- " "__UA_ADDR "\t1b, 4b \n"
- " "__UA_ADDR "\t2b, 4b \n"
+ " "__UA_ADDR "\t(1b + 0), 4b \n"
+ " "__UA_ADDR "\t(1b + 4), 4b \n"
+ " "__UA_ADDR "\t(2b + 0), 4b \n"
" .previous \n"
: "+r" (ret), "=&r" (val), "=" GCC_OFF_SMALL_ASM() (*uaddr)
: GCC_OFF_SMALL_ASM() (*uaddr), "Jr" (oldval), "Jr" (newval),
diff --git a/arch/mips/include/asm/local.h b/arch/mips/include/asm/local.h
index 02783e1..6b1936b 100644
--- a/arch/mips/include/asm/local.h
+++ b/arch/mips/include/asm/local.h
@@ -52,7 +52,9 @@ static __inline__ long local_add_return(long i, local_t * l)
__asm__ __volatile__(
" .set push \n"
" .set "MIPS_ISA_ARCH_LEVEL" \n"
- "1:" __LL "%1, %2 # local_add_return \n"
+ "1: # local_add_return \n"
+ __WAR_LLSC_MB
+ __LL "%1, %2 \n"
" addu %0, %1, %3 \n"
__SC "%0, %2 \n"
" beqz %0, 1b \n"
@@ -61,6 +63,7 @@ static __inline__ long local_add_return(long i, local_t * l)
: "=&r" (result), "=&r" (temp), "=m" (l->a.counter)
: "Ir" (i), "m" (l->a.counter)
: "memory");
+ __asm__ __volatile__(__WAR_LLSC_MB : : :"memory");
} else {
unsigned long flags;
@@ -99,7 +102,9 @@ static __inline__ long local_sub_return(long i, local_t * l)
__asm__ __volatile__(
" .set push \n"
" .set "MIPS_ISA_ARCH_LEVEL" \n"
- "1:" __LL "%1, %2 # local_sub_return \n"
+ "1: # local_sub_return \n"
+ __WAR_LLSC_MB
+ __LL "%1, %2 \n"
" subu %0, %1, %3 \n"
__SC "%0, %2 \n"
" beqz %0, 1b \n"
@@ -108,6 +113,7 @@ static __inline__ long local_sub_return(long i, local_t * l)
: "=&r" (result), "=&r" (temp), "=m" (l->a.counter)
: "Ir" (i), "m" (l->a.counter)
: "memory");
+ __asm__ __volatile__(__WAR_LLSC_MB : : :"memory");
} else {
unsigned long flags;
diff --git a/arch/mips/include/asm/pgtable.h b/arch/mips/include/asm/pgtable.h
index 57933fc..fcd3964 100644
--- a/arch/mips/include/asm/pgtable.h
+++ b/arch/mips/include/asm/pgtable.h
@@ -232,7 +232,9 @@ static inline void set_pte(pte_t *ptep, pte_t pteval)
" .set push \n"
" .set "MIPS_ISA_ARCH_LEVEL" \n"
" .set noreorder \n"
- "1:" __LL "%[tmp], %[buddy] \n"
+ "1: \n"
+ __WAR_LLSC_MB
+ __LL "%[tmp], %[buddy] \n"
" bnez %[tmp], 2f \n"
" or %[tmp], %[tmp], %[global] \n"
__SC "%[tmp], %[buddy] \n"
@@ -242,6 +244,7 @@ static inline void set_pte(pte_t *ptep, pte_t pteval)
" .set pop \n"
: [buddy] "+m" (buddy->pte), [tmp] "=&r" (tmp)
: [global] "r" (page_global));
+ __asm__ __volatile__(__WAR_LLSC_MB : : :"memory");
}
#else /* !CONFIG_SMP */
if (pte_none(*buddy))
diff --git a/arch/mips/kernel/syscall.c b/arch/mips/kernel/syscall.c
index 41a0db0..6fa3188 100644
--- a/arch/mips/kernel/syscall.c
+++ b/arch/mips/kernel/syscall.c
@@ -137,6 +137,7 @@ static inline int mips_atomic_set(unsigned long addr, unsigned long new)
" .set "MIPS_ISA_ARCH_LEVEL" \n"
" li %[err], 0 \n"
"1: \n"
+ __WAR_LLSC_MB
user_ll("%[old]", "(%[addr])")
" move %[tmp], %[new] \n"
"2: \n"
@@ -160,6 +161,7 @@ static inline int mips_atomic_set(unsigned long addr, unsigned long new)
[new] "r" (new),
[efault] "i" (-EFAULT)
: "memory");
+ __asm__ __volatile__(__WAR_LLSC_MB : : :"memory");
} else {
do {
preempt_disable();
diff --git a/arch/mips/loongson64/Platform b/arch/mips/loongson64/Platform
index 0fce460..3700dcf 100644
--- a/arch/mips/loongson64/Platform
+++ b/arch/mips/loongson64/Platform
@@ -23,6 +23,9 @@ ifdef CONFIG_CPU_LOONGSON2F_WORKAROUNDS
endif
cflags-$(CONFIG_CPU_LOONGSON3) += -Wa,--trap
+ifneq ($(call as-option,-Wa$(comma)-mfix-loongson3-llsc,),)
+ cflags-$(CONFIG_CPU_LOONGSON3) += -Wa$(comma)-mno-fix-loongson3-llsc
+endif
#
# binutils from v2.25 on and gcc starting from v4.9.0 treat -march=loongson3a
# as MIPS64 R2; older versions as just R1. This leaves the possibility open
diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c
index 0677142..74941b2 100644
--- a/arch/mips/mm/tlbex.c
+++ b/arch/mips/mm/tlbex.c
@@ -931,6 +931,8 @@ build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
* to mimic that here by taking a load/istream page
* fault.
*/
+ if (current_cpu_type() == CPU_LOONGSON3)
+ uasm_i_sync(p, 0);
UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0);
uasm_i_jr(p, ptr);
@@ -1555,6 +1557,7 @@ static void build_loongson3_tlb_refill_handler(void)
if (check_for_high_segbits) {
uasm_l_large_segbits_fault(&l, p);
+ uasm_i_sync(&p, 0);
UASM_i_LA(&p, K1, (unsigned long)tlb_do_page_fault_0);
uasm_i_jr(&p, K1);
uasm_i_nop(&p);
@@ -1645,6 +1648,8 @@ static void
iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr)
{
#ifdef CONFIG_SMP
+ if (current_cpu_type() == CPU_LOONGSON3)
+ uasm_i_sync(p, 0);
# ifdef CONFIG_PHYS_ADDR_T_64BIT
if (cpu_has_64bits)
uasm_i_lld(p, pte, 0, ptr);
@@ -2258,6 +2263,8 @@ static void build_r4000_tlb_load_handler(void)
#endif
uasm_l_nopage_tlbl(&l, p);
+ if (current_cpu_type() == CPU_LOONGSON3)
+ uasm_i_sync(&p, 0);
build_restore_work_registers(&p);
#ifdef CONFIG_CPU_MICROMIPS
if ((unsigned long)tlb_do_page_fault_0 & 1) {
@@ -2312,6 +2319,8 @@ static void build_r4000_tlb_store_handler(void)
#endif
uasm_l_nopage_tlbs(&l, p);
+ if (current_cpu_type() == CPU_LOONGSON3)
+ uasm_i_sync(&p, 0);
build_restore_work_registers(&p);
#ifdef CONFIG_CPU_MICROMIPS
if ((unsigned long)tlb_do_page_fault_1 & 1) {
@@ -2367,6 +2376,8 @@ static void build_r4000_tlb_modify_handler(void)
#endif
uasm_l_nopage_tlbm(&l, p);
+ if (current_cpu_type() == CPU_LOONGSON3)
+ uasm_i_sync(&p, 0);
build_restore_work_registers(&p);
#ifdef CONFIG_CPU_MICROMIPS
if ((unsigned long)tlb_do_page_fault_1 & 1) {
--
2.7.0
