Add copy_{to, from}_user() to machine check safe.
If copy fail due to hardware memory error, only the relevant processes are
affected, so killing the user process and isolate the user page with
hardware memory errors is a more reasonable choice than kernel panic.
Add new extable type EX_TYPE_UACCESS_MC which can be used for uaccess that
can be recovered from hardware memory errors.
Signed-off-by: Tong Tiangen <tongtiangen@xxxxxxxxxx>
---
arch/arm64/include/asm/asm-extable.h | 11 +++++++++++
arch/arm64/include/asm/asm-uaccess.h | 16 ++++++++++++++++
arch/arm64/lib/copy_from_user.S | 15 ++++++++++-----
arch/arm64/lib/copy_to_user.S | 25 +++++++++++++++++--------
4 files changed, 54 insertions(+), 13 deletions(-)
diff --git a/arch/arm64/include/asm/asm-extable.h b/arch/arm64/include/asm/asm-extable.h
index c39f2437e08e..8af4e7cc9578 100644
--- a/arch/arm64/include/asm/asm-extable.h
+++ b/arch/arm64/include/asm/asm-extable.h
@@ -8,6 +8,9 @@
#define EX_TYPE_UACCESS_ERR_ZERO 3
#define EX_TYPE_LOAD_UNALIGNED_ZEROPAD 4
+/* _MC indicates that can fixup from machine check errors */
+#define EX_TYPE_UACCESS_MC 5
+
#ifdef __ASSEMBLY__
#define __ASM_EXTABLE_RAW(insn, fixup, type, data) \
@@ -27,6 +30,14 @@
__ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_FIXUP, 0)
.endm
+/*
+ * Create an exception table entry for `insn`, which will branch to `fixup`
+ * when an unhandled fault(include sea fault) is taken.
+ */
+ .macro _asm_extable_uaccess_mc, insn, fixup
+ __ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_UACCESS_MC, 0)
+ .endm
+
/*
* Create an exception table entry for `insn` if `fixup` is provided. Otherwise
* do nothing.
diff --git a/arch/arm64/include/asm/asm-uaccess.h b/arch/arm64/include/asm/asm-uaccess.h
index 0557af834e03..bb17f0829042 100644
--- a/arch/arm64/include/asm/asm-uaccess.h
+++ b/arch/arm64/include/asm/asm-uaccess.h
@@ -92,4 +92,20 @@ alternative_else_nop_endif
_asm_extable 8888b,\l;
.endm
+
+ .macro user_ldp_mc l, reg1, reg2, addr, post_inc
+8888: ldtr \reg1, [\addr];
+8889: ldtr \reg2, [\addr, #8];
+ add \addr, \addr, \post_inc;
+
+ _asm_extable_uaccess_mc 8888b, \l;
+ _asm_extable_uaccess_mc 8889b, \l;
+ .endm
+
+ .macro user_ldst_mc l, inst, reg, addr, post_inc
+8888: \inst \reg, [\addr];
+ add \addr, \addr, \post_inc;
+
+ _asm_extable_uaccess_mc 8888b, \l;
+ .endm
#endif
diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S
index 34e317907524..e32c0747a5f1 100644
--- a/arch/arm64/lib/copy_from_user.S
+++ b/arch/arm64/lib/copy_from_user.S
@@ -21,7 +21,7 @@
*/
.macro ldrb1 reg, ptr, val
- user_ldst 9998f, ldtrb, \reg, \ptr, \val
+ user_ldst_mc 9998f, ldtrb, \reg, \ptr, \val
.endm
.macro strb1 reg, ptr, val
@@ -29,7 +29,7 @@
.endm
.macro ldrh1 reg, ptr, val
- user_ldst 9997f, ldtrh, \reg, \ptr, \val
+ user_ldst_mc 9997f, ldtrh, \reg, \ptr, \val
.endm
.macro strh1 reg, ptr, val
@@ -37,7 +37,7 @@
.endm
.macro ldr1 reg, ptr, val
- user_ldst 9997f, ldtr, \reg, \ptr, \val
+ user_ldst_mc 9997f, ldtr, \reg, \ptr, \val
.endm
.macro str1 reg, ptr, val
@@ -45,7 +45,7 @@
.endm
.macro ldp1 reg1, reg2, ptr, val
- user_ldp 9997f, \reg1, \reg2, \ptr, \val
+ user_ldp_mc 9997f, \reg1, \reg2, \ptr, \val
.endm
.macro stp1 reg1, reg2, ptr, val
@@ -54,6 +54,7 @@
end .req x5
srcin .req x15
+esr .req x16
SYM_FUNC_START(__arch_copy_from_user)
add end, x0, x2
mov srcin, x1
@@ -62,7 +63,11 @@ SYM_FUNC_START(__arch_copy_from_user)
ret
// Exception fixups
-9997: cmp dst, dstin
+9997: mrs esr, esr_el1 // Check exception first
+ and esr, esr, #ESR_ELx_FSC
+ cmp esr, #ESR_ELx_FSC_EXTABT
+ b.eq 9998f
+ cmp dst, dstin
b.ne 9998f
// Before being absolutely sure we couldn't copy anything, try harder
USER(9998f, ldtrb tmp1w, [srcin])
diff --git a/arch/arm64/lib/copy_to_user.S b/arch/arm64/lib/copy_to_user.S
index 802231772608..afb53e45a21f 100644
--- a/arch/arm64/lib/copy_to_user.S
+++ b/arch/arm64/lib/copy_to_user.S
@@ -20,31 +20,35 @@
* x0 - bytes not copied
*/
.macro ldrb1 reg, ptr, val
- ldrb \reg, [\ptr], \val
+ 1000: ldrb \reg, [\ptr], \val
+ _asm_extable_uaccess_mc 1000b, 9998f;
.endm
.macro strb1 reg, ptr, val
- user_ldst 9998f, sttrb, \reg, \ptr, \val
+ user_ldst_mc 9998f, sttrb, \reg, \ptr, \val
.endm
.macro ldrh1 reg, ptr, val
- ldrh \reg, [\ptr], \val
+ 1001: ldrh \reg, [\ptr], \val
+ _asm_extable_uaccess_mc 1001b, 9998f;
.endm
.macro strh1 reg, ptr, val
- user_ldst 9997f, sttrh, \reg, \ptr, \val
+ user_ldst_mc 9997f, sttrh, \reg, \ptr, \val
.endm
.macro ldr1 reg, ptr, val
- ldr \reg, [\ptr], \val
+ 1002: ldr \reg, [\ptr], \val
+ _asm_extable_uaccess_mc 1002b, 9998f;
.endm
.macro str1 reg, ptr, val
- user_ldst 9997f, sttr, \reg, \ptr, \val
+ user_ldst_mc 9997f, sttr, \reg, \ptr, \val
.endm
.macro ldp1 reg1, reg2, ptr, val
- ldp \reg1, \reg2, [\ptr], \val
+ 1003: ldp \reg1, \reg2, [\ptr], \val
+ _asm_extable_uaccess_mc 1003b, 9998f;
.endm
.macro stp1 reg1, reg2, ptr, val
@@ -53,6 +57,7 @@
end .req x5
srcin .req x15
+esr .req x16
SYM_FUNC_START(__arch_copy_to_user)
add end, x0, x2
mov srcin, x1
@@ -61,7 +66,11 @@ SYM_FUNC_START(__arch_copy_to_user)
ret
// Exception fixups
-9997: cmp dst, dstin
+9997: mrs esr, esr_el1 // Check exception first
+ and esr, esr, #ESR_ELx_FSC
+ cmp esr, #ESR_ELx_FSC_EXTABT
+ b.eq 9998f
+ cmp dst, dstin
b.ne 9998f
// Before being absolutely sure we couldn't copy anything, try harder
ldrb tmp1w, [srcin]
--
2.18.0.huawei.25
