s390/extable: add dedicated uaccess handler

This is more or less a combination of commit 2e77a62cb3a6 ("arm64:
extable: add a dedicated uaccess handler") and commit 4b5305decc84
("x86/extable: Extend extable functionality").

To describe the problem that needs to solved let's cite the full arm64
commit message:

------
For inline assembly, we place exception fixups out-of-line in the
`.fixup` section such that these are out of the way of the fast path.
This has a few drawbacks:

* Since the fixup code is anonymous, backtraces will symbolize fixups
  as offsets from the nearest prior symbol, currently
  `__entry_tramp_text_end`. This is confusing, and painful to debug
  without access to the relevant vmlinux.

* Since the exception handler adjusts the PC to execute the fixup, and
  the fixup uses a direct branch back into the function it fixes,
  backtraces of fixups miss the original function. This is confusing,
  and violates requirements for RELIABLE_STACKTRACE (and therefore
  LIVEPATCH).

* Inline assembly and associated fixups are generated from templates,
  and we have many copies of logically identical fixups which only
  differ in which specific registers are written to and which address
  is branched to at the end of the fixup. This is potentially wasteful
  of I-cache resources, and makes it hard to add additional logic to
  fixups without significant bloat.

This patch address all three concerns for inline uaccess fixups by
adding a dedicated exception handler which updates registers in
exception context and subsequent returns back into the function which
faulted, removing the need for fixups specialized to each faulting
instruction.

Other than backtracing, there should be no functional change as a result
of this patch.
------

Acked-by: Alexander Gordeev <agordeev@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>

3 files changed, 43 insertions, 18 deletions

diff --git a/arch/s390/include/asm/asm-extable.h b/arch/s390/include/asm/asm-extable.h
index 61484a5f1209..fb62df5e16a2 100644
--- a/arch/s390/include/asm/asm-extable.h
+++ b/arch/s390/include/asm/asm-extable.h
@@ -8,6 +8,7 @@
 #define EX_TYPE_NONE	0
 #define EX_TYPE_FIXUP	1
 #define EX_TYPE_BPF	2
+#define EX_TYPE_UACCESS	3
 
 #define __EX_TABLE(_section, _fault, _target, _type)			\
 	stringify_in_c(.section	_section,"a";)				\
@@ -18,9 +19,35 @@
 	stringify_in_c(.short	0;)					\
 	stringify_in_c(.previous)
 
+#define __EX_TABLE_UA(_section, _fault, _target, _type, _reg)		\
+	stringify_in_c(.section _section,"a";)				\
+	stringify_in_c(.align	4;)					\
+	stringify_in_c(.long	(_fault) - .;)				\
+	stringify_in_c(.long	(_target) - .;)				\
+	stringify_in_c(.short	(_type);)				\
+	stringify_in_c(.macro extable_reg reg;)				\
+	stringify_in_c(.set found, 0;)					\
+	stringify_in_c(.set regnr, 0;)					\
+	stringify_in_c(.irp rs,r0,r1,r2,r3,r4,r5,r6,r7,r8,r9,r10,r11,r12,r13,r14,r15;) \
+	stringify_in_c(.ifc "\reg", "%%\rs";)				\
+	stringify_in_c(.set found, 1;)					\
+	stringify_in_c(.short regnr;)					\
+	stringify_in_c(.endif;)						\
+	stringify_in_c(.set regnr, regnr+1;)				\
+	stringify_in_c(.endr;)						\
+	stringify_in_c(.ifne (found != 1);)				\
+	stringify_in_c(.error "extable_reg: bad register argument";)	\
+	stringify_in_c(.endif;)						\
+	stringify_in_c(.endm;)						\
+	stringify_in_c(extable_reg _reg;)				\
+	stringify_in_c(.purgem extable_reg;)				\
+	stringify_in_c(.previous)
+
 #define EX_TABLE(_fault, _target)					\
 	__EX_TABLE(__ex_table, _fault, _target, EX_TYPE_FIXUP)
 #define EX_TABLE_AMODE31(_fault, _target)				\
 	__EX_TABLE(.amode31.ex_table, _fault, _target, EX_TYPE_FIXUP)
+#define EX_TABLE_UA(_fault, _target, _reg)				\
+	__EX_TABLE_UA(__ex_table, _fault, _target, EX_TYPE_UACCESS, _reg)
 
 #endif /* __ASM_EXTABLE_H */
diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h
index 2c029ee89b7c..44b18800721a 100644
--- a/arch/s390/include/asm/uaccess.h
+++ b/arch/s390/include/asm/uaccess.h
@@ -81,14 +81,10 @@ union oac {
 		"0:	mvcos	%[_to],%[_from],%[_size]\n"		\
 		"1:	xr	%[rc],%[rc]\n"				\
 		"2:\n"							\
-		".pushsection .fixup, \"ax\"\n"				\
-		"3:	lhi	%[rc],%[retval]\n"			\
-		"	jg	2b\n"					\
-		".popsection\n"						\
-		EX_TABLE(0b,3b) EX_TABLE(1b,3b)				\
+		EX_TABLE_UA(0b,2b,%[rc]) EX_TABLE_UA(1b,2b,%[rc])	\
 		: [rc] "=&d" (__rc), [_to] "+Q" (*(to))			\
 		: [_size] "d" (size), [_from] "Q" (*(from)),		\
-		  [retval] "K" (-EFAULT), [spec] "d" (oac_spec.val)	\
+		  [spec] "d" (oac_spec.val)				\
 		: "cc", "0");						\
 	__rc;								\
 })
@@ -295,13 +291,9 @@ int __noreturn __put_kernel_bad(void);
 		"0:   " insn "  %2,%1\n"				\
 		"1:	xr	%0,%0\n"				\
 		"2:\n"							\
-		".pushsection .fixup, \"ax\"\n"				\
-		"3:	lhi	%0,%3\n"				\
-		"	jg	2b\n"					\
-		".popsection\n"						\
-		EX_TABLE(0b,3b) EX_TABLE(1b,3b)				\
+		EX_TABLE_UA(0b,2b,%0) EX_TABLE_UA(1b,2b,%0)		\
 		: "=d" (__rc), "+Q" (*(to))				\
-		: "d" (val), "K" (-EFAULT)				\
+		: "d" (val)						\
 		: "cc");						\
 	__rc;								\
 })
@@ -342,13 +334,9 @@ int __noreturn __get_kernel_bad(void);
 		"0:   " insn "  %1,%2\n"				\
 		"1:	xr	%0,%0\n"				\
 		"2:\n"							\
-		".pushsection .fixup, \"ax\"\n"				\
-		"3:	lhi	%0,%3\n"				\
-		"	jg	2b\n"					\
-		".popsection\n"						\
-		EX_TABLE(0b,3b) EX_TABLE(1b,3b)				\
+		EX_TABLE_UA(0b,2b,%0) EX_TABLE_UA(1b,2b,%0)		\
 		: "=d" (__rc), "+d" (val)				\
-		: "Q" (*(from)), "K" (-EFAULT)				\
+		: "Q" (*(from))						\
 		: "cc");						\
 	__rc;								\
 })
diff --git a/arch/s390/mm/extable.c b/arch/s390/mm/extable.c
index ac6b736ac883..8ac8ad2474a0 100644
--- a/arch/s390/mm/extable.c
+++ b/arch/s390/mm/extable.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 
 #include <linux/extable.h>
+#include <linux/errno.h>
 #include <linux/panic.h>
 #include <asm/asm-extable.h>
 #include <asm/extable.h>
@@ -23,6 +24,13 @@ static bool ex_handler_fixup(const struct exception_table_entry *ex, struct pt_r
 	return true;
 }
 
+static bool ex_handler_uaccess(const struct exception_table_entry *ex, struct pt_regs *regs)
+{
+	regs->gprs[ex->data] = -EFAULT;
+	regs->psw.addr = extable_fixup(ex);
+	return true;
+}
+
 bool fixup_exception(struct pt_regs *regs)
 {
 	const struct exception_table_entry *ex;
@@ -35,6 +43,8 @@ bool fixup_exception(struct pt_regs *regs)
 		return ex_handler_fixup(ex, regs);
 	case EX_TYPE_BPF:
 		return ex_handler_bpf(ex, regs);
+	case EX_TYPE_UACCESS:
+		return ex_handler_uaccess(ex, regs);
 	}
 	panic("invalid exception table entry");
 }