summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/arm64/include/asm/asm-extable.h32
-rw-r--r--arch/arm64/include/asm/extable.h19
-rw-r--r--arch/arm64/kernel/vmlinux.lds.S2
-rw-r--r--arch/arm64/mm/extable.c29
-rw-r--r--arch/arm64/net/bpf_jit_comp.c7
-rw-r--r--scripts/sorttable.c30
6 files changed, 96 insertions, 23 deletions
diff --git a/arch/arm64/include/asm/asm-extable.h b/arch/arm64/include/asm/asm-extable.h
index 986b4c0d4792..5ee748edaef1 100644
--- a/arch/arm64/include/asm/asm-extable.h
+++ b/arch/arm64/include/asm/asm-extable.h
@@ -2,13 +2,19 @@
#ifndef __ASM_ASM_EXTABLE_H
#define __ASM_ASM_EXTABLE_H
+#define EX_TYPE_NONE 0
+#define EX_TYPE_FIXUP 1
+#define EX_TYPE_BPF 2
+
#ifdef __ASSEMBLY__
-#define __ASM_EXTABLE_RAW(insn, fixup) \
- .pushsection __ex_table, "a"; \
- .align 3; \
- .long ((insn) - .); \
- .long ((fixup) - .); \
+#define __ASM_EXTABLE_RAW(insn, fixup, type, data) \
+ .pushsection __ex_table, "a"; \
+ .align 2; \
+ .long ((insn) - .); \
+ .long ((fixup) - .); \
+ .short (type); \
+ .short (data); \
.popsection;
/*
@@ -16,7 +22,7 @@
* when an unhandled fault is taken.
*/
.macro _asm_extable, insn, fixup
- __ASM_EXTABLE_RAW(\insn, \fixup)
+ __ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_FIXUP, 0)
.endm
/*
@@ -33,15 +39,17 @@
#include <linux/stringify.h>
-#define __ASM_EXTABLE_RAW(insn, fixup) \
- ".pushsection __ex_table, \"a\"\n" \
- ".align 3\n" \
- ".long ((" insn ") - .)\n" \
- ".long ((" fixup ") - .)\n" \
+#define __ASM_EXTABLE_RAW(insn, fixup, type, data) \
+ ".pushsection __ex_table, \"a\"\n" \
+ ".align 2\n" \
+ ".long ((" insn ") - .)\n" \
+ ".long ((" fixup ") - .)\n" \
+ ".short (" type ")\n" \
+ ".short (" data ")\n" \
".popsection\n"
#define _ASM_EXTABLE(insn, fixup) \
- __ASM_EXTABLE_RAW(#insn, #fixup)
+ __ASM_EXTABLE_RAW(#insn, #fixup, __stringify(EX_TYPE_FIXUP), "0")
#endif /* __ASSEMBLY__ */
diff --git a/arch/arm64/include/asm/extable.h b/arch/arm64/include/asm/extable.h
index 1859b9fd566f..8b300dd28def 100644
--- a/arch/arm64/include/asm/extable.h
+++ b/arch/arm64/include/asm/extable.h
@@ -18,10 +18,21 @@
struct exception_table_entry
{
int insn, fixup;
+ short type, data;
};
#define ARCH_HAS_RELATIVE_EXTABLE
+#define swap_ex_entry_fixup(a, b, tmp, delta) \
+do { \
+ (a)->fixup = (b)->fixup + (delta); \
+ (b)->fixup = (tmp).fixup - (delta); \
+ (a)->type = (b)->type; \
+ (b)->type = (tmp).type; \
+ (a)->data = (b)->data; \
+ (b)->data = (tmp).data; \
+} while (0)
+
static inline bool in_bpf_jit(struct pt_regs *regs)
{
if (!IS_ENABLED(CONFIG_BPF_JIT))
@@ -32,12 +43,12 @@ static inline bool in_bpf_jit(struct pt_regs *regs)
}
#ifdef CONFIG_BPF_JIT
-bool arm64_bpf_fixup_exception(const struct exception_table_entry *ex,
- struct pt_regs *regs);
+bool ex_handler_bpf(const struct exception_table_entry *ex,
+ struct pt_regs *regs);
#else /* !CONFIG_BPF_JIT */
static inline
-bool arm64_bpf_fixup_exception(const struct exception_table_entry *ex,
- struct pt_regs *regs)
+bool ex_handler_bpf(const struct exception_table_entry *ex,
+ struct pt_regs *regs)
{
return false;
}
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index f6b1a88245db..fa8a8e8ddcfd 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -57,7 +57,7 @@
#define SBSS_ALIGN 0
#endif
-#define RO_EXCEPTION_TABLE_ALIGN 8
+#define RO_EXCEPTION_TABLE_ALIGN 4
#define RUNTIME_DISCARD_EXIT
#include <asm-generic/vmlinux.lds.h>
diff --git a/arch/arm64/mm/extable.c b/arch/arm64/mm/extable.c
index dba3d59f3eca..c2951b963335 100644
--- a/arch/arm64/mm/extable.c
+++ b/arch/arm64/mm/extable.c
@@ -6,6 +6,24 @@
#include <linux/extable.h>
#include <linux/uaccess.h>
+#include <asm/asm-extable.h>
+
+typedef bool (*ex_handler_t)(const struct exception_table_entry *,
+ struct pt_regs *);
+
+static inline unsigned long
+get_ex_fixup(const struct exception_table_entry *ex)
+{
+ return ((unsigned long)&ex->fixup + ex->fixup);
+}
+
+static bool ex_handler_fixup(const struct exception_table_entry *ex,
+ struct pt_regs *regs)
+{
+ regs->pc = get_ex_fixup(ex);
+ return true;
+}
+
bool fixup_exception(struct pt_regs *regs)
{
const struct exception_table_entry *ex;
@@ -14,9 +32,12 @@ bool fixup_exception(struct pt_regs *regs)
if (!ex)
return false;
- if (in_bpf_jit(regs))
- return arm64_bpf_fixup_exception(ex, regs);
+ switch (ex->type) {
+ case EX_TYPE_FIXUP:
+ return ex_handler_fixup(ex, regs);
+ case EX_TYPE_BPF:
+ return ex_handler_bpf(ex, regs);
+ }
- regs->pc = (unsigned long)&ex->fixup + ex->fixup;
- return true;
+ BUG();
}
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index 956c841ef346..7df7345e60d8 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -13,6 +13,7 @@
#include <linux/printk.h>
#include <linux/slab.h>
+#include <asm/asm-extable.h>
#include <asm/byteorder.h>
#include <asm/cacheflush.h>
#include <asm/debug-monitors.h>
@@ -358,8 +359,8 @@ static void build_epilogue(struct jit_ctx *ctx)
#define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0)
#define BPF_FIXUP_REG_MASK GENMASK(31, 27)
-bool arm64_bpf_fixup_exception(const struct exception_table_entry *ex,
- struct pt_regs *regs)
+bool ex_handler_bpf(const struct exception_table_entry *ex,
+ struct pt_regs *regs)
{
off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup);
int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup);
@@ -412,6 +413,8 @@ static int add_exception_handler(const struct bpf_insn *insn,
ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) |
FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg);
+ ex->type = EX_TYPE_BPF;
+
ctx->exentry_idx++;
return 0;
}
diff --git a/scripts/sorttable.c b/scripts/sorttable.c
index 6ee4fa882919..ee95bb47a50d 100644
--- a/scripts/sorttable.c
+++ b/scripts/sorttable.c
@@ -231,6 +231,34 @@ static void sort_relative_table(char *extab_image, int image_size)
}
}
+static void arm64_sort_relative_table(char *extab_image, int image_size)
+{
+ int i = 0;
+
+ while (i < image_size) {
+ uint32_t *loc = (uint32_t *)(extab_image + i);
+
+ w(r(loc) + i, loc);
+ w(r(loc + 1) + i + 4, loc + 1);
+ /* Don't touch the fixup type or data */
+
+ i += sizeof(uint32_t) * 3;
+ }
+
+ qsort(extab_image, image_size / 12, 12, compare_relative_table);
+
+ i = 0;
+ while (i < image_size) {
+ uint32_t *loc = (uint32_t *)(extab_image + i);
+
+ w(r(loc) - i, loc);
+ w(r(loc + 1) - (i + 4), loc + 1);
+ /* Don't touch the fixup type or data */
+
+ i += sizeof(uint32_t) * 3;
+ }
+}
+
static void x86_sort_relative_table(char *extab_image, int image_size)
{
int i = 0;
@@ -343,6 +371,8 @@ static int do_file(char const *const fname, void *addr)
custom_sort = s390_sort_relative_table;
break;
case EM_AARCH64:
+ custom_sort = arm64_sort_relative_table;
+ break;
case EM_PARISC:
case EM_PPC:
case EM_PPC64:
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-09 23:00:57 +0000