From 3384c7c7641b44987e35eadbc9df6c16a0520159 Mon Sep 17 00:00:00 2001 From: Vadim Fedorenko Date: Fri, 10 Sep 2021 01:04:09 +0300 Subject: selftests/bpf: Test new __sk_buff field hwtstamp Analogous to the gso_segs selftests introduced in commit d9ff286a0f59 ("bpf: allow BPF programs access skb_shared_info->gso_segs field"). Signed-off-by: Vadim Fedorenko Signed-off-by: Daniel Borkmann Acked-by: Martin KaFai Lau Link: https://lore.kernel.org/bpf/20210909220409.8804-3-vfedorenko@novek.ru --- lib/test_bpf.c | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index 830a18ecffc8..0018d51b93b0 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -8800,6 +8800,7 @@ static __init struct sk_buff *build_test_skb(void) skb_shinfo(skb[0])->gso_type |= SKB_GSO_DODGY; skb_shinfo(skb[0])->gso_segs = 0; skb_shinfo(skb[0])->frag_list = skb[1]; + skb_shinfo(skb[0])->hwtstamps.hwtstamp = 1000; /* adjust skb[0]'s len */ skb[0]->len += skb[1]->len; -- cgit v1.2.3 From c2a228d69cef802cf6bfd773c84f8419d2e2acf9 Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Tue, 14 Sep 2021 11:18:29 +0200 Subject: bpf/tests: Allow different number of runs per test case This patch allows a test cast to specify the number of runs to use. For compatibility with existing test case definitions, the default value 0 is interpreted as MAX_TESTRUNS. A reduced number of runs is useful for complex test programs where 1000 runs may take a very long time. Instead of reducing what is tested, one can instead reduce the number of times the test is run. Signed-off-by: Johan Almbladh Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20210914091842.4186267-2-johan.almbladh@anyfinetworks.com --- lib/test_bpf.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index 0018d51b93b0..a406cbb2e34d 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -80,6 +80,7 @@ struct bpf_test { int expected_errcode; /* used when FLAG_EXPECTED_FAIL is set in the aux */ __u8 frag_data[MAX_DATA]; int stack_depth; /* for eBPF only, since tests don't call verifier */ + int nr_testruns; /* Custom run count, defaults to MAX_TESTRUNS if 0 */ }; /* Large test cases need separate allocation and fill handler. */ @@ -8631,6 +8632,9 @@ static int run_one(const struct bpf_prog *fp, struct bpf_test *test) { int err_cnt = 0, i, runs = MAX_TESTRUNS; + if (test->nr_testruns) + runs = min(test->nr_testruns, MAX_TESTRUNS); + for (i = 0; i < MAX_SUBTESTS; i++) { void *data; u64 duration; -- cgit v1.2.3 From 4bc354138d553bc48dc1fb1e184d50a524e6c20f Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Tue, 14 Sep 2021 11:18:30 +0200 Subject: bpf/tests: Reduce memory footprint of test suite The test suite used to call any fill_helper callbacks to generate eBPF program data for all test cases at once. This caused ballooning memory requirements as more extensive test cases were added. Now the each fill_helper is called before the test is run and the allocated memory released afterwards, before the next test case is processed. Signed-off-by: Johan Almbladh Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20210914091842.4186267-3-johan.almbladh@anyfinetworks.com --- lib/test_bpf.c | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index a406cbb2e34d..c865fd584adb 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -8694,8 +8694,6 @@ static __init int find_test_index(const char *test_name) static __init int prepare_bpf_tests(void) { - int i; - if (test_id >= 0) { /* * if a test_id was specified, use test_range to @@ -8739,23 +8737,11 @@ static __init int prepare_bpf_tests(void) } } - for (i = 0; i < ARRAY_SIZE(tests); i++) { - if (tests[i].fill_helper && - tests[i].fill_helper(&tests[i]) < 0) - return -ENOMEM; - } - return 0; } static __init void destroy_bpf_tests(void) { - int i; - - for (i = 0; i < ARRAY_SIZE(tests); i++) { - if (tests[i].fill_helper) - kfree(tests[i].u.ptr.insns); - } } static bool exclude_test(int test_id) @@ -8960,7 +8946,19 @@ static __init int test_bpf(void) pr_info("#%d %s ", i, tests[i].descr); + if (tests[i].fill_helper && + tests[i].fill_helper(&tests[i]) < 0) { + pr_cont("FAIL to prog_fill\n"); + continue; + } + fp = generate_filter(i, &err); + + if (tests[i].fill_helper) { + kfree(tests[i].u.ptr.insns); + tests[i].u.ptr.insns = NULL; + } + if (fp == NULL) { if (err == 0) { pass_cnt++; -- cgit v1.2.3 From 68c956fe741757b760aa00fca8725c5651f5f77a Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Tue, 14 Sep 2021 11:18:31 +0200 Subject: bpf/tests: Add exhaustive tests of ALU shift values This patch adds a set of tests for ALU64 and ALU32 shift operations to verify correctness for all possible values of the shift value. Mainly intended for JIT testing. Signed-off-by: Johan Almbladh Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20210914091842.4186267-4-johan.almbladh@anyfinetworks.com --- lib/test_bpf.c | 260 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 260 insertions(+) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index c865fd584adb..d599c2a8c860 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -497,6 +497,168 @@ static int bpf_fill_long_jmp(struct bpf_test *self) return 0; } +static int __bpf_ld_imm64(struct bpf_insn insns[2], u8 reg, s64 imm64) +{ + struct bpf_insn tmp[] = {BPF_LD_IMM64(reg, imm64)}; + + memcpy(insns, tmp, sizeof(tmp)); + return 2; +} + +/* Test an ALU shift operation for all valid shift values */ +static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op, + u8 mode, bool alu32) +{ + static const s64 regs[] = { + 0x0123456789abcdefLL, /* dword > 0, word < 0 */ + 0xfedcba9876543210LL, /* dowrd < 0, word > 0 */ + 0xfedcba0198765432LL, /* dowrd < 0, word < 0 */ + 0x0123458967abcdefLL, /* dword > 0, word > 0 */ + }; + int bits = alu32 ? 32 : 64; + int len = (2 + 7 * bits) * ARRAY_SIZE(regs) + 3; + struct bpf_insn *insn; + int imm, k; + int i = 0; + + insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL); + if (!insn) + return -ENOMEM; + + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); + + for (k = 0; k < ARRAY_SIZE(regs); k++) { + s64 reg = regs[k]; + + i += __bpf_ld_imm64(&insn[i], R3, reg); + + for (imm = 0; imm < bits; imm++) { + u64 val; + + /* Perform operation */ + insn[i++] = BPF_ALU64_REG(BPF_MOV, R1, R3); + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R2, imm); + if (alu32) { + if (mode == BPF_K) + insn[i++] = BPF_ALU32_IMM(op, R1, imm); + else + insn[i++] = BPF_ALU32_REG(op, R1, R2); + switch (op) { + case BPF_LSH: + val = (u32)reg << imm; + break; + case BPF_RSH: + val = (u32)reg >> imm; + break; + case BPF_ARSH: + val = (u32)reg >> imm; + if (imm > 0 && (reg & 0x80000000)) + val |= ~(u32)0 << (32 - imm); + break; + } + } else { + if (mode == BPF_K) + insn[i++] = BPF_ALU64_IMM(op, R1, imm); + else + insn[i++] = BPF_ALU64_REG(op, R1, R2); + switch (op) { + case BPF_LSH: + val = (u64)reg << imm; + break; + case BPF_RSH: + val = (u64)reg >> imm; + break; + case BPF_ARSH: + val = (u64)reg >> imm; + if (imm > 0 && reg < 0) + val |= ~(u64)0 << (64 - imm); + break; + } + } + + /* + * When debugging a JIT that fails this test, one + * can write the immediate value to R0 here to find + * out which operand values that fail. + */ + + /* Load reference and check the result */ + i += __bpf_ld_imm64(&insn[i], R4, val); + insn[i++] = BPF_JMP_REG(BPF_JEQ, R1, R4, 1); + insn[i++] = BPF_EXIT_INSN(); + } + } + + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); + insn[i++] = BPF_EXIT_INSN(); + + self->u.ptr.insns = insn; + self->u.ptr.len = len; + BUG_ON(i > len); + + return 0; +} + +static int bpf_fill_alu_lsh_imm(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_LSH, BPF_K, false); +} + +static int bpf_fill_alu_rsh_imm(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_RSH, BPF_K, false); +} + +static int bpf_fill_alu_arsh_imm(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_K, false); +} + +static int bpf_fill_alu_lsh_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_LSH, BPF_X, false); +} + +static int bpf_fill_alu_rsh_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_RSH, BPF_X, false); +} + +static int bpf_fill_alu_arsh_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, false); +} + +static int bpf_fill_alu32_lsh_imm(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_LSH, BPF_K, true); +} + +static int bpf_fill_alu32_rsh_imm(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_RSH, BPF_K, true); +} + +static int bpf_fill_alu32_arsh_imm(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_K, true); +} + +static int bpf_fill_alu32_lsh_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_LSH, BPF_X, true); +} + +static int bpf_fill_alu32_rsh_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_RSH, BPF_X, true); +} + +static int bpf_fill_alu32_arsh_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, true); +} + static struct bpf_test tests[] = { { "TAX", @@ -8414,6 +8576,104 @@ static struct bpf_test tests[] = { {}, { { 0, 2 } }, }, + /* Exhaustive test of ALU64 shift operations */ + { + "ALU64_LSH_K: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu_lsh_imm, + }, + { + "ALU64_RSH_K: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu_rsh_imm, + }, + { + "ALU64_ARSH_K: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu_arsh_imm, + }, + { + "ALU64_LSH_X: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu_lsh_reg, + }, + { + "ALU64_RSH_X: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu_rsh_reg, + }, + { + "ALU64_ARSH_X: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu_arsh_reg, + }, + /* Exhaustive test of ALU32 shift operations */ + { + "ALU32_LSH_K: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_lsh_imm, + }, + { + "ALU32_RSH_K: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_rsh_imm, + }, + { + "ALU32_ARSH_K: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_arsh_imm, + }, + { + "ALU32_LSH_X: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_lsh_reg, + }, + { + "ALU32_RSH_X: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_rsh_reg, + }, + { + "ALU32_ARSH_X: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_arsh_reg, + }, }; static struct net_device dev; -- cgit v1.2.3 From 9298e63eafea1ebe235919dbbbfc20c1c25000f8 Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Tue, 14 Sep 2021 11:18:32 +0200 Subject: bpf/tests: Add exhaustive tests of ALU operand magnitudes This patch adds a set of tests for ALU64 and ALU32 arithmetic and bitwise logical operations to verify correctness for all possible magnitudes of the register and immediate operands. Mainly intended for JIT testing. The patch introduces a pattern generator that can be used to drive extensive tests of different kinds of operations. It is parameterized to allow tuning of the operand combinations to test. Signed-off-by: Johan Almbladh Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20210914091842.4186267-5-johan.almbladh@anyfinetworks.com --- lib/test_bpf.c | 772 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 772 insertions(+) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index d599c2a8c860..228e681e84b7 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -659,6 +659,451 @@ static int bpf_fill_alu32_arsh_reg(struct bpf_test *self) return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, true); } +/* + * Common operand pattern generator for exhaustive power-of-two magnitudes + * tests. The block size parameters can be adjusted to increase/reduce the + * number of combinatons tested and thereby execution speed and memory + * footprint. + */ + +static inline s64 value(int msb, int delta, int sign) +{ + return sign * (1LL << msb) + delta; +} + +static int __bpf_fill_pattern(struct bpf_test *self, void *arg, + int dbits, int sbits, int block1, int block2, + int (*emit)(struct bpf_test*, void*, + struct bpf_insn*, s64, s64)) +{ + static const int sgn[][2] = {{1, 1}, {1, -1}, {-1, 1}, {-1, -1}}; + struct bpf_insn *insns; + int di, si, bt, db, sb; + int count, len, k; + int extra = 1 + 2; + int i = 0; + + /* Total number of iterations for the two pattern */ + count = (dbits - 1) * (sbits - 1) * block1 * block1 * ARRAY_SIZE(sgn); + count += (max(dbits, sbits) - 1) * block2 * block2 * ARRAY_SIZE(sgn); + + /* Compute the maximum number of insns and allocate the buffer */ + len = extra + count * (*emit)(self, arg, NULL, 0, 0); + insns = kmalloc_array(len, sizeof(*insns), GFP_KERNEL); + if (!insns) + return -ENOMEM; + + /* Add head instruction(s) */ + insns[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); + + /* + * Pattern 1: all combinations of power-of-two magnitudes and sign, + * and with a block of contiguous values around each magnitude. + */ + for (di = 0; di < dbits - 1; di++) /* Dst magnitudes */ + for (si = 0; si < sbits - 1; si++) /* Src magnitudes */ + for (k = 0; k < ARRAY_SIZE(sgn); k++) /* Sign combos */ + for (db = -(block1 / 2); + db < (block1 + 1) / 2; db++) + for (sb = -(block1 / 2); + sb < (block1 + 1) / 2; sb++) { + s64 dst, src; + + dst = value(di, db, sgn[k][0]); + src = value(si, sb, sgn[k][1]); + i += (*emit)(self, arg, + &insns[i], + dst, src); + } + /* + * Pattern 2: all combinations for a larger block of values + * for each power-of-two magnitude and sign, where the magnitude is + * the same for both operands. + */ + for (bt = 0; bt < max(dbits, sbits) - 1; bt++) /* Magnitude */ + for (k = 0; k < ARRAY_SIZE(sgn); k++) /* Sign combos */ + for (db = -(block2 / 2); db < (block2 + 1) / 2; db++) + for (sb = -(block2 / 2); + sb < (block2 + 1) / 2; sb++) { + s64 dst, src; + + dst = value(bt % dbits, db, sgn[k][0]); + src = value(bt % sbits, sb, sgn[k][1]); + i += (*emit)(self, arg, &insns[i], + dst, src); + } + + /* Append tail instructions */ + insns[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); + insns[i++] = BPF_EXIT_INSN(); + BUG_ON(i > len); + + self->u.ptr.insns = insns; + self->u.ptr.len = i; + + return 0; +} + +/* + * Block size parameters used in pattern tests below. une as needed to + * increase/reduce the number combinations tested, see following examples. + * block values per operand MSB + * ---------------------------------------- + * 0 none + * 1 (1 << MSB) + * 2 (1 << MSB) + [-1, 0] + * 3 (1 << MSB) + [-1, 0, 1] + */ +#define PATTERN_BLOCK1 1 +#define PATTERN_BLOCK2 5 + +/* Number of test runs for a pattern test */ +#define NR_PATTERN_RUNS 1 + +/* + * Exhaustive tests of ALU operations for all combinations of power-of-two + * magnitudes of the operands, both for positive and negative values. The + * test is designed to verify e.g. the JMP and JMP32 operations for JITs that + * emit different code depending on the magnitude of the immediate value. + */ + +static bool __bpf_alu_result(u64 *res, u64 v1, u64 v2, u8 op) +{ + *res = 0; + switch (op) { + case BPF_MOV: + *res = v2; + break; + case BPF_AND: + *res = v1 & v2; + break; + case BPF_OR: + *res = v1 | v2; + break; + case BPF_XOR: + *res = v1 ^ v2; + break; + case BPF_ADD: + *res = v1 + v2; + break; + case BPF_SUB: + *res = v1 - v2; + break; + case BPF_MUL: + *res = v1 * v2; + break; + case BPF_DIV: + if (v2 == 0) + return false; + *res = div64_u64(v1, v2); + break; + case BPF_MOD: + if (v2 == 0) + return false; + div64_u64_rem(v1, v2, res); + break; + } + return true; +} + +static int __bpf_emit_alu64_imm(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 imm) +{ + int op = *(int *)arg; + int i = 0; + u64 res; + + if (!insns) + return 7; + + if (__bpf_alu_result(&res, dst, (s32)imm, op)) { + i += __bpf_ld_imm64(&insns[i], R1, dst); + i += __bpf_ld_imm64(&insns[i], R3, res); + insns[i++] = BPF_ALU64_IMM(op, R1, imm); + insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); + insns[i++] = BPF_EXIT_INSN(); + } + + return i; +} + +static int __bpf_emit_alu32_imm(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 imm) +{ + int op = *(int *)arg; + int i = 0; + u64 res; + + if (!insns) + return 7; + + if (__bpf_alu_result(&res, (u32)dst, (u32)imm, op)) { + i += __bpf_ld_imm64(&insns[i], R1, dst); + i += __bpf_ld_imm64(&insns[i], R3, (u32)res); + insns[i++] = BPF_ALU32_IMM(op, R1, imm); + insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); + insns[i++] = BPF_EXIT_INSN(); + } + + return i; +} + +static int __bpf_emit_alu64_reg(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 src) +{ + int op = *(int *)arg; + int i = 0; + u64 res; + + if (!insns) + return 9; + + if (__bpf_alu_result(&res, dst, src, op)) { + i += __bpf_ld_imm64(&insns[i], R1, dst); + i += __bpf_ld_imm64(&insns[i], R2, src); + i += __bpf_ld_imm64(&insns[i], R3, res); + insns[i++] = BPF_ALU64_REG(op, R1, R2); + insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); + insns[i++] = BPF_EXIT_INSN(); + } + + return i; +} + +static int __bpf_emit_alu32_reg(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 src) +{ + int op = *(int *)arg; + int i = 0; + u64 res; + + if (!insns) + return 9; + + if (__bpf_alu_result(&res, (u32)dst, (u32)src, op)) { + i += __bpf_ld_imm64(&insns[i], R1, dst); + i += __bpf_ld_imm64(&insns[i], R2, src); + i += __bpf_ld_imm64(&insns[i], R3, (u32)res); + insns[i++] = BPF_ALU32_REG(op, R1, R2); + insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); + insns[i++] = BPF_EXIT_INSN(); + } + + return i; +} + +static int __bpf_fill_alu64_imm(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 32, + PATTERN_BLOCK1, PATTERN_BLOCK2, + &__bpf_emit_alu64_imm); +} + +static int __bpf_fill_alu32_imm(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 32, + PATTERN_BLOCK1, PATTERN_BLOCK2, + &__bpf_emit_alu32_imm); +} + +static int __bpf_fill_alu64_reg(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 64, + PATTERN_BLOCK1, PATTERN_BLOCK2, + &__bpf_emit_alu64_reg); +} + +static int __bpf_fill_alu32_reg(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 64, + PATTERN_BLOCK1, PATTERN_BLOCK2, + &__bpf_emit_alu32_reg); +} + +/* ALU64 immediate operations */ +static int bpf_fill_alu64_mov_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_MOV); +} + +static int bpf_fill_alu64_and_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_AND); +} + +static int bpf_fill_alu64_or_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_OR); +} + +static int bpf_fill_alu64_xor_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_XOR); +} + +static int bpf_fill_alu64_add_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_ADD); +} + +static int bpf_fill_alu64_sub_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_SUB); +} + +static int bpf_fill_alu64_mul_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_MUL); +} + +static int bpf_fill_alu64_div_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_DIV); +} + +static int bpf_fill_alu64_mod_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_MOD); +} + +/* ALU32 immediate operations */ +static int bpf_fill_alu32_mov_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_MOV); +} + +static int bpf_fill_alu32_and_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_AND); +} + +static int bpf_fill_alu32_or_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_OR); +} + +static int bpf_fill_alu32_xor_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_XOR); +} + +static int bpf_fill_alu32_add_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_ADD); +} + +static int bpf_fill_alu32_sub_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_SUB); +} + +static int bpf_fill_alu32_mul_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_MUL); +} + +static int bpf_fill_alu32_div_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_DIV); +} + +static int bpf_fill_alu32_mod_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_MOD); +} + +/* ALU64 register operations */ +static int bpf_fill_alu64_mov_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_MOV); +} + +static int bpf_fill_alu64_and_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_AND); +} + +static int bpf_fill_alu64_or_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_OR); +} + +static int bpf_fill_alu64_xor_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_XOR); +} + +static int bpf_fill_alu64_add_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_ADD); +} + +static int bpf_fill_alu64_sub_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_SUB); +} + +static int bpf_fill_alu64_mul_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_MUL); +} + +static int bpf_fill_alu64_div_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_DIV); +} + +static int bpf_fill_alu64_mod_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_MOD); +} + +/* ALU32 register operations */ +static int bpf_fill_alu32_mov_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_MOV); +} + +static int bpf_fill_alu32_and_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_AND); +} + +static int bpf_fill_alu32_or_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_OR); +} + +static int bpf_fill_alu32_xor_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_XOR); +} + +static int bpf_fill_alu32_add_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_ADD); +} + +static int bpf_fill_alu32_sub_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_SUB); +} + +static int bpf_fill_alu32_mul_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_MUL); +} + +static int bpf_fill_alu32_div_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_DIV); +} + +static int bpf_fill_alu32_mod_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_MOD); +} + static struct bpf_test tests[] = { { "TAX", @@ -8674,6 +9119,333 @@ static struct bpf_test tests[] = { { { 0, 1 } }, .fill_helper = bpf_fill_alu32_arsh_reg, }, + /* ALU64 immediate magnitudes */ + { + "ALU64_MOV_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mov_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_AND_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_and_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_OR_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_or_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_XOR_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_xor_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_ADD_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_add_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_SUB_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_sub_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_MUL_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mul_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_DIV_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_div_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_MOD_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mod_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* ALU32 immediate magnitudes */ + { + "ALU32_MOV_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mov_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_AND_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_and_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_OR_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_or_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_XOR_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_xor_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_ADD_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_add_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_SUB_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_sub_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_MUL_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mul_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_DIV_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_div_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_MOD_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mod_imm, + }, + /* ALU64 register magnitudes */ + { + "ALU64_MOV_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mov_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_AND_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_and_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_OR_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_or_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_XOR_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_xor_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_ADD_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_add_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_SUB_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_sub_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_MUL_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mul_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_DIV_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_div_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_MOD_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mod_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* ALU32 register magnitudes */ + { + "ALU32_MOV_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mov_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_AND_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_and_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_OR_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_or_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_XOR_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_xor_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_ADD_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_add_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_SUB_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_sub_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_MUL_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mul_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_DIV_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_div_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_MOD_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mod_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, }; static struct net_device dev; -- cgit v1.2.3 From a5a36544de38057b8e8de8fb6b2bcd9c102640f4 Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Tue, 14 Sep 2021 11:18:33 +0200 Subject: bpf/tests: Add exhaustive tests of JMP operand magnitudes This patch adds a set of tests for conditional JMP and JMP32 operations to verify correctness for all possible magnitudes of the immediate and register operands. Mainly intended for JIT testing. Signed-off-by: Johan Almbladh Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20210914091842.4186267-6-johan.almbladh@anyfinetworks.com --- lib/test_bpf.c | 779 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 779 insertions(+) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index 228e681e84b7..fb27a25e2c0c 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -1104,6 +1104,384 @@ static int bpf_fill_alu32_mod_reg(struct bpf_test *self) return __bpf_fill_alu32_reg(self, BPF_MOD); } + +/* + * Exhaustive tests of JMP operations for all combinations of power-of-two + * magnitudes of the operands, both for positive and negative values. The + * test is designed to verify e.g. the JMP and JMP32 operations for JITs that + * emit different code depending on the magnitude of the immediate value. + */ + +static bool __bpf_match_jmp_cond(s64 v1, s64 v2, u8 op) +{ + switch (op) { + case BPF_JSET: + return !!(v1 & v2); + case BPF_JEQ: + return v1 == v2; + case BPF_JNE: + return v1 != v2; + case BPF_JGT: + return (u64)v1 > (u64)v2; + case BPF_JGE: + return (u64)v1 >= (u64)v2; + case BPF_JLT: + return (u64)v1 < (u64)v2; + case BPF_JLE: + return (u64)v1 <= (u64)v2; + case BPF_JSGT: + return v1 > v2; + case BPF_JSGE: + return v1 >= v2; + case BPF_JSLT: + return v1 < v2; + case BPF_JSLE: + return v1 <= v2; + } + return false; +} + +static int __bpf_emit_jmp_imm(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 imm) +{ + int op = *(int *)arg; + + if (insns) { + bool match = __bpf_match_jmp_cond(dst, (s32)imm, op); + int i = 0; + + insns[i++] = BPF_ALU32_IMM(BPF_MOV, R0, match); + + i += __bpf_ld_imm64(&insns[i], R1, dst); + insns[i++] = BPF_JMP_IMM(op, R1, imm, 1); + if (!match) + insns[i++] = BPF_JMP_IMM(BPF_JA, 0, 0, 1); + insns[i++] = BPF_EXIT_INSN(); + + return i; + } + + return 5 + 1; +} + +static int __bpf_emit_jmp32_imm(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 imm) +{ + int op = *(int *)arg; + + if (insns) { + bool match = __bpf_match_jmp_cond((s32)dst, (s32)imm, op); + int i = 0; + + i += __bpf_ld_imm64(&insns[i], R1, dst); + insns[i++] = BPF_JMP32_IMM(op, R1, imm, 1); + if (!match) + insns[i++] = BPF_JMP_IMM(BPF_JA, 0, 0, 1); + insns[i++] = BPF_EXIT_INSN(); + + return i; + } + + return 5; +} + +static int __bpf_emit_jmp_reg(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 src) +{ + int op = *(int *)arg; + + if (insns) { + bool match = __bpf_match_jmp_cond(dst, src, op); + int i = 0; + + i += __bpf_ld_imm64(&insns[i], R1, dst); + i += __bpf_ld_imm64(&insns[i], R2, src); + insns[i++] = BPF_JMP_REG(op, R1, R2, 1); + if (!match) + insns[i++] = BPF_JMP_IMM(BPF_JA, 0, 0, 1); + insns[i++] = BPF_EXIT_INSN(); + + return i; + } + + return 7; +} + +static int __bpf_emit_jmp32_reg(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 src) +{ + int op = *(int *)arg; + + if (insns) { + bool match = __bpf_match_jmp_cond((s32)dst, (s32)src, op); + int i = 0; + + i += __bpf_ld_imm64(&insns[i], R1, dst); + i += __bpf_ld_imm64(&insns[i], R2, src); + insns[i++] = BPF_JMP32_REG(op, R1, R2, 1); + if (!match) + insns[i++] = BPF_JMP_IMM(BPF_JA, 0, 0, 1); + insns[i++] = BPF_EXIT_INSN(); + + return i; + } + + return 7; +} + +static int __bpf_fill_jmp_imm(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 32, + PATTERN_BLOCK1, PATTERN_BLOCK2, + &__bpf_emit_jmp_imm); +} + +static int __bpf_fill_jmp32_imm(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 32, + PATTERN_BLOCK1, PATTERN_BLOCK2, + &__bpf_emit_jmp32_imm); +} + +static int __bpf_fill_jmp_reg(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 64, + PATTERN_BLOCK1, PATTERN_BLOCK2, + &__bpf_emit_jmp_reg); +} + +static int __bpf_fill_jmp32_reg(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 64, + PATTERN_BLOCK1, PATTERN_BLOCK2, + &__bpf_emit_jmp32_reg); +} + +/* JMP immediate tests */ +static int bpf_fill_jmp_jset_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JSET); +} + +static int bpf_fill_jmp_jeq_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JEQ); +} + +static int bpf_fill_jmp_jne_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JNE); +} + +static int bpf_fill_jmp_jgt_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JGT); +} + +static int bpf_fill_jmp_jge_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JGE); +} + +static int bpf_fill_jmp_jlt_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JLT); +} + +static int bpf_fill_jmp_jle_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JLE); +} + +static int bpf_fill_jmp_jsgt_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JSGT); +} + +static int bpf_fill_jmp_jsge_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JSGE); +} + +static int bpf_fill_jmp_jslt_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JSLT); +} + +static int bpf_fill_jmp_jsle_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JSLE); +} + +/* JMP32 immediate tests */ +static int bpf_fill_jmp32_jset_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JSET); +} + +static int bpf_fill_jmp32_jeq_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JEQ); +} + +static int bpf_fill_jmp32_jne_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JNE); +} + +static int bpf_fill_jmp32_jgt_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JGT); +} + +static int bpf_fill_jmp32_jge_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JGE); +} + +static int bpf_fill_jmp32_jlt_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JLT); +} + +static int bpf_fill_jmp32_jle_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JLE); +} + +static int bpf_fill_jmp32_jsgt_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JSGT); +} + +static int bpf_fill_jmp32_jsge_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JSGE); +} + +static int bpf_fill_jmp32_jslt_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JSLT); +} + +static int bpf_fill_jmp32_jsle_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JSLE); +} + +/* JMP register tests */ +static int bpf_fill_jmp_jset_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JSET); +} + +static int bpf_fill_jmp_jeq_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JEQ); +} + +static int bpf_fill_jmp_jne_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JNE); +} + +static int bpf_fill_jmp_jgt_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JGT); +} + +static int bpf_fill_jmp_jge_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JGE); +} + +static int bpf_fill_jmp_jlt_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JLT); +} + +static int bpf_fill_jmp_jle_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JLE); +} + +static int bpf_fill_jmp_jsgt_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JSGT); +} + +static int bpf_fill_jmp_jsge_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JSGE); +} + +static int bpf_fill_jmp_jslt_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JSLT); +} + +static int bpf_fill_jmp_jsle_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JSLE); +} + +/* JMP32 register tests */ +static int bpf_fill_jmp32_jset_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JSET); +} + +static int bpf_fill_jmp32_jeq_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JEQ); +} + +static int bpf_fill_jmp32_jne_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JNE); +} + +static int bpf_fill_jmp32_jgt_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JGT); +} + +static int bpf_fill_jmp32_jge_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JGE); +} + +static int bpf_fill_jmp32_jlt_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JLT); +} + +static int bpf_fill_jmp32_jle_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JLE); +} + +static int bpf_fill_jmp32_jsgt_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JSGT); +} + +static int bpf_fill_jmp32_jsge_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JSGE); +} + +static int bpf_fill_jmp32_jslt_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JSLT); +} + +static int bpf_fill_jmp32_jsle_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JSLE); +} + + static struct bpf_test tests[] = { { "TAX", @@ -9281,6 +9659,7 @@ static struct bpf_test tests[] = { { }, { { 0, 1 } }, .fill_helper = bpf_fill_alu32_mod_imm, + .nr_testruns = NR_PATTERN_RUNS, }, /* ALU64 register magnitudes */ { @@ -9446,6 +9825,406 @@ static struct bpf_test tests[] = { .fill_helper = bpf_fill_alu32_mod_reg, .nr_testruns = NR_PATTERN_RUNS, }, + /* JMP immediate magnitudes */ + { + "JMP_JSET_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jset_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JEQ_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jeq_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JNE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jne_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JGT_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jgt_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JGE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jge_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JLT_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jlt_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JLE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jle_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JSGT_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jsgt_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JSGE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jsge_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JSLT_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jslt_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JSLE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jsle_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* JMP register magnitudes */ + { + "JMP_JSET_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jset_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JEQ_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jeq_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JNE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jne_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JGT_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jgt_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JGE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jge_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JLT_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jlt_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JLE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jle_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JSGT_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jsgt_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JSGE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jsge_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JSLT_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jslt_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JSLE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jsle_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* JMP32 immediate magnitudes */ + { + "JMP32_JSET_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jset_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JEQ_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jeq_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JNE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jne_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JGT_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jgt_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JGE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jge_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JLT_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jlt_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JLE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jle_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JSGT_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jsgt_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JSGE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jsge_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JSLT_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jslt_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JSLE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jsle_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* JMP32 register magnitudes */ + { + "JMP32_JSET_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jset_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JEQ_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jeq_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JNE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jne_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JGT_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jgt_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JGE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jge_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JLT_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jlt_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JLE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jle_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JSGT_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jsgt_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JSGE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jsge_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JSLT_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jslt_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JSLE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jsle_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, }; static struct net_device dev; -- cgit v1.2.3 From a7d2e752e52050fcdf0c50cf343488891a8efd5b Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Tue, 14 Sep 2021 11:18:34 +0200 Subject: bpf/tests: Add staggered JMP and JMP32 tests This patch adds a new type of jump test where the program jumps forwards and backwards with increasing offset. It mainly tests JITs where a relative jump may generate different JITed code depending on the offset size, read MIPS. Signed-off-by: Johan Almbladh Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20210914091842.4186267-7-johan.almbladh@anyfinetworks.com --- lib/test_bpf.c | 829 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 829 insertions(+) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index fb27a25e2c0c..7b3d801275e8 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -1481,6 +1481,426 @@ static int bpf_fill_jmp32_jsle_reg(struct bpf_test *self) return __bpf_fill_jmp32_reg(self, BPF_JSLE); } +/* + * Set up a sequence of staggered jumps, forwards and backwards with + * increasing offset. This tests the conversion of relative jumps to + * JITed native jumps. On some architectures, for example MIPS, a large + * PC-relative jump offset may overflow the immediate field of the native + * conditional branch instruction, triggering a conversion to use an + * absolute jump instead. Since this changes the jump offsets, another + * offset computation pass is necessary, and that may in turn trigger + * another branch conversion. This jump sequence is particularly nasty + * in that regard. + * + * The sequence generation is parameterized by size and jump type. + * The size must be even, and the expected result is always size + 1. + * Below is an example with size=8 and result=9. + * + * ________________________Start + * R0 = 0 + * R1 = r1 + * R2 = r2 + * ,------- JMP +4 * 3______________Preamble: 4 insns + * ,----------|-ind 0- if R0 != 7 JMP 8 * 3 + 1 <--------------------. + * | | R0 = 8 | + * | | JMP +7 * 3 ------------------------. + * | ,--------|-----1- if R0 != 5 JMP 7 * 3 + 1 <--------------. | | + * | | | R0 = 6 | | | + * | | | JMP +5 * 3 ------------------. | | + * | | ,------|-----2- if R0 != 3 JMP 6 * 3 + 1 <--------. | | | | + * | | | | R0 = 4 | | | | | + * | | | | JMP +3 * 3 ------------. | | | | + * | | | ,----|-----3- if R0 != 1 JMP 5 * 3 + 1 <--. | | | | | | + * | | | | | R0 = 2 | | | | | | | + * | | | | | JMP +1 * 3 ------. | | | | | | + * | | | | ,--t=====4> if R0 != 0 JMP 4 * 3 + 1 1 2 3 4 5 6 7 8 loc + * | | | | | R0 = 1 -1 +2 -3 +4 -5 +6 -7 +8 off + * | | | | | JMP -2 * 3 ---' | | | | | | | + * | | | | | ,------5- if R0 != 2 JMP 3 * 3 + 1 <-----' | | | | | | + * | | | | | | R0 = 3 | | | | | | + * | | | | | | JMP -4 * 3 ---------' | | | | | + * | | | | | | ,----6- if R0 != 4 JMP 2 * 3 + 1 <-----------' | | | | + * | | | | | | | R0 = 5 | | | | + * | | | | | | | JMP -6 * 3 ---------------' | | | + * | | | | | | | ,--7- if R0 != 6 JMP 1 * 3 + 1 <-----------------' | | + * | | | | | | | | R0 = 7 | | + * | | Error | | | JMP -8 * 3 ---------------------' | + * | | paths | | | ,8- if R0 != 8 JMP 0 * 3 + 1 <-----------------------' + * | | | | | | | | | R0 = 9__________________Sequence: 3 * size - 1 insns + * `-+-+-+-+-+-+-+-+-> EXIT____________________Return: 1 insn + * + */ + +/* The maximum size parameter */ +#define MAX_STAGGERED_JMP_SIZE ((0x7fff / 3) & ~1) + +/* We use a reduced number of iterations to get a reasonable execution time */ +#define NR_STAGGERED_JMP_RUNS 10 + +static int __bpf_fill_staggered_jumps(struct bpf_test *self, + const struct bpf_insn *jmp, + u64 r1, u64 r2) +{ + int size = self->test[0].result - 1; + int len = 4 + 3 * (size + 1); + struct bpf_insn *insns; + int off, ind; + + insns = kmalloc_array(len, sizeof(*insns), GFP_KERNEL); + if (!insns) + return -ENOMEM; + + /* Preamble */ + insns[0] = BPF_ALU64_IMM(BPF_MOV, R0, 0); + insns[1] = BPF_ALU64_IMM(BPF_MOV, R1, r1); + insns[2] = BPF_ALU64_IMM(BPF_MOV, R2, r2); + insns[3] = BPF_JMP_IMM(BPF_JA, 0, 0, 3 * size / 2); + + /* Sequence */ + for (ind = 0, off = size; ind <= size; ind++, off -= 2) { + struct bpf_insn *ins = &insns[4 + 3 * ind]; + int loc; + + if (off == 0) + off--; + + loc = abs(off); + ins[0] = BPF_JMP_IMM(BPF_JNE, R0, loc - 1, + 3 * (size - ind) + 1); + ins[1] = BPF_ALU64_IMM(BPF_MOV, R0, loc); + ins[2] = *jmp; + ins[2].off = 3 * (off - 1); + } + + /* Return */ + insns[len - 1] = BPF_EXIT_INSN(); + + self->u.ptr.insns = insns; + self->u.ptr.len = len; + + return 0; +} + +/* 64-bit unconditional jump */ +static int bpf_fill_staggered_ja(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JA, 0, 0, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0, 0); +} + +/* 64-bit immediate jumps */ +static int bpf_fill_staggered_jeq_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JEQ, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); +} + +static int bpf_fill_staggered_jne_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JNE, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 4321, 0); +} + +static int bpf_fill_staggered_jset_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSET, R1, 0x82, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0x86, 0); +} + +static int bpf_fill_staggered_jgt_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JGT, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0x80000000, 0); +} + +static int bpf_fill_staggered_jge_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JGE, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); +} + +static int bpf_fill_staggered_jlt_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JLT, R1, 0x80000000, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); +} + +static int bpf_fill_staggered_jle_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JLE, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); +} + +static int bpf_fill_staggered_jsgt_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSGT, R1, -2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -1, 0); +} + +static int bpf_fill_staggered_jsge_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSGE, R1, -2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -2, 0); +} + +static int bpf_fill_staggered_jslt_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSLT, R1, -1, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -2, 0); +} + +static int bpf_fill_staggered_jsle_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSLE, R1, -1, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -1, 0); +} + +/* 64-bit register jumps */ +static int bpf_fill_staggered_jeq_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JEQ, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); +} + +static int bpf_fill_staggered_jne_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JNE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 4321, 1234); +} + +static int bpf_fill_staggered_jset_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JSET, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0x86, 0x82); +} + +static int bpf_fill_staggered_jgt_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JGT, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0x80000000, 1234); +} + +static int bpf_fill_staggered_jge_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JGE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); +} + +static int bpf_fill_staggered_jlt_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JLT, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0x80000000); +} + +static int bpf_fill_staggered_jle_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JLE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); +} + +static int bpf_fill_staggered_jsgt_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JSGT, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -1, -2); +} + +static int bpf_fill_staggered_jsge_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JSGE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -2, -2); +} + +static int bpf_fill_staggered_jslt_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JSLT, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -2, -1); +} + +static int bpf_fill_staggered_jsle_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JSLE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -1, -1); +} + +/* 32-bit immediate jumps */ +static int bpf_fill_staggered_jeq32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JEQ, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); +} + +static int bpf_fill_staggered_jne32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JNE, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 4321, 0); +} + +static int bpf_fill_staggered_jset32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSET, R1, 0x82, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0x86, 0); +} + +static int bpf_fill_staggered_jgt32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JGT, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0x80000000, 0); +} + +static int bpf_fill_staggered_jge32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JGE, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); +} + +static int bpf_fill_staggered_jlt32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JLT, R1, 0x80000000, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); +} + +static int bpf_fill_staggered_jle32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JLE, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); +} + +static int bpf_fill_staggered_jsgt32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSGT, R1, -2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -1, 0); +} + +static int bpf_fill_staggered_jsge32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSGE, R1, -2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -2, 0); +} + +static int bpf_fill_staggered_jslt32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSLT, R1, -1, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -2, 0); +} + +static int bpf_fill_staggered_jsle32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSLE, R1, -1, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -1, 0); +} + +/* 32-bit register jumps */ +static int bpf_fill_staggered_jeq32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JEQ, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); +} + +static int bpf_fill_staggered_jne32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JNE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 4321, 1234); +} + +static int bpf_fill_staggered_jset32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSET, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0x86, 0x82); +} + +static int bpf_fill_staggered_jgt32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JGT, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0x80000000, 1234); +} + +static int bpf_fill_staggered_jge32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JGE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); +} + +static int bpf_fill_staggered_jlt32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JLT, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0x80000000); +} + +static int bpf_fill_staggered_jle32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JLE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); +} + +static int bpf_fill_staggered_jsgt32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSGT, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -1, -2); +} + +static int bpf_fill_staggered_jsge32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSGE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -2, -2); +} + +static int bpf_fill_staggered_jslt32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSLT, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -2, -1); +} + +static int bpf_fill_staggered_jsle32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSLE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -1, -1); +} + static struct bpf_test tests[] = { { @@ -10225,6 +10645,415 @@ static struct bpf_test tests[] = { .fill_helper = bpf_fill_jmp32_jsle_reg, .nr_testruns = NR_PATTERN_RUNS, }, + /* Staggered jump sequences, immediate */ + { + "Staggered jumps: JMP_JA", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_ja, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JEQ_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jeq_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JNE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jne_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSET_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jset_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JGT_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jgt_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JGE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jge_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JLT_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jlt_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JLE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jle_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSGT_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsgt_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSGE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsge_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSLT_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jslt_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSLE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsle_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + /* Staggered jump sequences, register */ + { + "Staggered jumps: JMP_JEQ_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jeq_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JNE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jne_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSET_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jset_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JGT_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jgt_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JGE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jge_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JLT_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jlt_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JLE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jle_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSGT_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsgt_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSGE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsge_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSLT_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jslt_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSLE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsle_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + /* Staggered jump sequences, JMP32 immediate */ + { + "Staggered jumps: JMP32_JEQ_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jeq32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JNE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jne32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSET_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jset32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JGT_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jgt32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JGE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jge32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JLT_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jlt32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JLE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jle32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSGT_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsgt32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSGE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsge32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSLT_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jslt32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSLE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsle32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + /* Staggered jump sequences, JMP32 register */ + { + "Staggered jumps: JMP32_JEQ_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jeq32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JNE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jne32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSET_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jset32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JGT_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jgt32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JGE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jge32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JLT_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jlt32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JLE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jle32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSGT_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsgt32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSGE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsge32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSLT_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jslt32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSLE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsle32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, }; static struct net_device dev; -- cgit v1.2.3 From 2e807611945c2d36e25d10bc6f932e5f9943deea Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Tue, 14 Sep 2021 11:18:35 +0200 Subject: bpf/tests: Add exhaustive test of LD_IMM64 immediate magnitudes This patch adds a test for the 64-bit immediate load, a two-instruction operation, to verify correctness for all possible magnitudes of the immediate operand. Mainly intended for JIT testing. Signed-off-by: Johan Almbladh Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20210914091842.4186267-8-johan.almbladh@anyfinetworks.com --- lib/test_bpf.c | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index 7b3d801275e8..8aef42cfb5fe 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -1104,6 +1104,60 @@ static int bpf_fill_alu32_mod_reg(struct bpf_test *self) return __bpf_fill_alu32_reg(self, BPF_MOD); } +/* + * Test the two-instruction 64-bit immediate load operation for all + * power-of-two magnitudes of the immediate operand. For each MSB, a block + * of immediate values centered around the power-of-two MSB are tested, + * both for positive and negative values. The test is designed to verify + * the operation for JITs that emit different code depending on the magnitude + * of the immediate value. This is often the case if the native instruction + * immediate field width is narrower than 32 bits. + */ +static int bpf_fill_ld_imm64(struct bpf_test *self) +{ + int block = 64; /* Increase for more tests per MSB position */ + int len = 3 + 8 * 63 * block * 2; + struct bpf_insn *insn; + int bit, adj, sign; + int i = 0; + + insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL); + if (!insn) + return -ENOMEM; + + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); + + for (bit = 0; bit <= 62; bit++) { + for (adj = -block / 2; adj < block / 2; adj++) { + for (sign = -1; sign <= 1; sign += 2) { + s64 imm = sign * ((1LL << bit) + adj); + + /* Perform operation */ + i += __bpf_ld_imm64(&insn[i], R1, imm); + + /* Load reference */ + insn[i++] = BPF_ALU32_IMM(BPF_MOV, R2, imm); + insn[i++] = BPF_ALU32_IMM(BPF_MOV, R3, + (u32)(imm >> 32)); + insn[i++] = BPF_ALU64_IMM(BPF_LSH, R3, 32); + insn[i++] = BPF_ALU64_REG(BPF_OR, R2, R3); + + /* Check result */ + insn[i++] = BPF_JMP_REG(BPF_JEQ, R1, R2, 1); + insn[i++] = BPF_EXIT_INSN(); + } + } + } + + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); + insn[i++] = BPF_EXIT_INSN(); + + self->u.ptr.insns = insn; + self->u.ptr.len = len; + BUG_ON(i != len); + + return 0; +} /* * Exhaustive tests of JMP operations for all combinations of power-of-two @@ -10245,6 +10299,15 @@ static struct bpf_test tests[] = { .fill_helper = bpf_fill_alu32_mod_reg, .nr_testruns = NR_PATTERN_RUNS, }, + /* LD_IMM64 immediate magnitudes */ + { + "LD_IMM64: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_ld_imm64, + }, /* JMP immediate magnitudes */ { "JMP_JSET_K: all immediate value magnitudes", -- cgit v1.2.3 From 27cc6dac6ec816cc31be9031edbee3e519234471 Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Tue, 14 Sep 2021 11:18:36 +0200 Subject: bpf/tests: Add test case flag for verifier zero-extension This patch adds a new flag to indicate that the verified did insert zero-extensions, even though the verifier is not being run for any of the tests. Signed-off-by: Johan Almbladh Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20210914091842.4186267-9-johan.almbladh@anyfinetworks.com --- lib/test_bpf.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index 8aef42cfb5fe..0c1c8cccf254 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -52,6 +52,7 @@ #define FLAG_NO_DATA BIT(0) #define FLAG_EXPECTED_FAIL BIT(1) #define FLAG_SKB_FRAG BIT(2) +#define FLAG_VERIFIER_ZEXT BIT(3) enum { CLASSIC = BIT(6), /* Old BPF instructions only. */ @@ -11280,6 +11281,8 @@ static struct bpf_prog *generate_filter(int which, int *err) fp->type = BPF_PROG_TYPE_SOCKET_FILTER; memcpy(fp->insnsi, fptr, fp->len * sizeof(struct bpf_insn)); fp->aux->stack_depth = tests[which].stack_depth; + fp->aux->verifier_zext = !!(tests[which].aux & + FLAG_VERIFIER_ZEXT); /* We cannot error here as we don't need type compatibility * checks. -- cgit v1.2.3 From d4ff9ee2dc0bbbdba204e215c8b6bf58f5773994 Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Tue, 14 Sep 2021 11:18:37 +0200 Subject: bpf/tests: Add JMP tests with small offsets This patch adds a set of tests for JMP to verify that the JITed jump offset is calculated correctly. We pretend that the verifier has inserted any zero extensions to make the jump-over operations JIT to one instruction each, in order to control the exact JITed jump offset. Signed-off-by: Johan Almbladh Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20210914091842.4186267-10-johan.almbladh@anyfinetworks.com --- lib/test_bpf.c | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index 0c1c8cccf254..2eb1d0e4aff3 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -10709,6 +10709,77 @@ static struct bpf_test tests[] = { .fill_helper = bpf_fill_jmp32_jsle_reg, .nr_testruns = NR_PATTERN_RUNS, }, + /* Short relative jumps */ + { + "Short relative jump: offset=0", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_JMP_IMM(BPF_JEQ, R0, 0, 0), + BPF_EXIT_INSN(), + BPF_ALU32_IMM(BPF_MOV, R0, -1), + }, + INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, + { }, + { { 0, 0 } }, + }, + { + "Short relative jump: offset=1", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_JMP_IMM(BPF_JEQ, R0, 0, 1), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_EXIT_INSN(), + BPF_ALU32_IMM(BPF_MOV, R0, -1), + }, + INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, + { }, + { { 0, 0 } }, + }, + { + "Short relative jump: offset=2", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_JMP_IMM(BPF_JEQ, R0, 0, 2), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_EXIT_INSN(), + BPF_ALU32_IMM(BPF_MOV, R0, -1), + }, + INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, + { }, + { { 0, 0 } }, + }, + { + "Short relative jump: offset=3", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_JMP_IMM(BPF_JEQ, R0, 0, 3), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_EXIT_INSN(), + BPF_ALU32_IMM(BPF_MOV, R0, -1), + }, + INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, + { }, + { { 0, 0 } }, + }, + { + "Short relative jump: offset=4", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_JMP_IMM(BPF_JEQ, R0, 0, 4), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_EXIT_INSN(), + BPF_ALU32_IMM(BPF_MOV, R0, -1), + }, + INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, + { }, + { { 0, 0 } }, + }, /* Staggered jump sequences, immediate */ { "Staggered jumps: JMP_JA", -- cgit v1.2.3 From c4df4559db8447cdae15254a713f7fd5d4cee3ab Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Tue, 14 Sep 2021 11:18:38 +0200 Subject: bpf/tests: Add JMP tests with degenerate conditional This patch adds a set of tests for JMP and JMP32 operations where the branch decision is know at JIT time. Mainly testing JIT behaviour. Signed-off-by: Johan Almbladh Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20210914091842.4186267-11-johan.almbladh@anyfinetworks.com --- lib/test_bpf.c | 229 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 229 insertions(+) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index 2eb1d0e4aff3..2aa1d0d3c685 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -10709,6 +10709,235 @@ static struct bpf_test tests[] = { .fill_helper = bpf_fill_jmp32_jsle_reg, .nr_testruns = NR_PATTERN_RUNS, }, + /* Conditional jumps with constant decision */ + { + "JMP_JSET_K: imm = 0 -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_IMM(BPF_JSET, R1, 0, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP_JLT_K: imm = 0 -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_IMM(BPF_JLT, R1, 0, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP_JGE_K: imm = 0 -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_IMM(BPF_JGE, R1, 0, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP_JGT_K: imm = 0xffffffff -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_IMM(BPF_JGT, R1, U32_MAX, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP_JLE_K: imm = 0xffffffff -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_IMM(BPF_JLE, R1, U32_MAX, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP32_JSGT_K: imm = 0x7fffffff -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP32_IMM(BPF_JSGT, R1, S32_MAX, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP32_JSGE_K: imm = -0x80000000 -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP32_IMM(BPF_JSGE, R1, S32_MIN, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP32_JSLT_K: imm = -0x80000000 -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP32_IMM(BPF_JSLT, R1, S32_MIN, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP32_JSLE_K: imm = 0x7fffffff -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP32_IMM(BPF_JSLE, R1, S32_MAX, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP_JEQ_X: dst = src -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JEQ, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP_JGE_X: dst = src -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JGE, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP_JLE_X: dst = src -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JLE, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP_JSGE_X: dst = src -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JSGE, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP_JSLE_X: dst = src -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JSLE, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP_JNE_X: dst = src -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JNE, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP_JGT_X: dst = src -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JGT, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP_JLT_X: dst = src -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JLT, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP_JSGT_X: dst = src -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JSGT, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP_JSLT_X: dst = src -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JSLT, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, /* Short relative jumps */ { "Short relative jump: offset=0", -- cgit v1.2.3 From f1517eb790f97c1326016eb164a33a64d4d4fb7a Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Tue, 14 Sep 2021 11:18:39 +0200 Subject: bpf/tests: Expand branch conversion JIT test This patch expands the branch conversion test introduced by 66e5eb84 ("bpf, tests: Add branch conversion JIT test"). The test now includes a JMP with maximum eBPF offset. This triggers branch conversion for the 64-bit MIPS JIT. Additional variants are also added for cases when the branch is taken or not taken. Signed-off-by: Johan Almbladh Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20210914091842.4186267-12-johan.almbladh@anyfinetworks.com --- lib/test_bpf.c | 125 +++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 91 insertions(+), 34 deletions(-) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index 2aa1d0d3c685..dd6bb0044097 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -463,47 +463,79 @@ static int bpf_fill_stxdw(struct bpf_test *self) return __bpf_fill_stxdw(self, BPF_DW); } -static int bpf_fill_long_jmp(struct bpf_test *self) +static int __bpf_ld_imm64(struct bpf_insn insns[2], u8 reg, s64 imm64) { - unsigned int len = BPF_MAXINSNS; - struct bpf_insn *insn; + struct bpf_insn tmp[] = {BPF_LD_IMM64(reg, imm64)}; + + memcpy(insns, tmp, sizeof(tmp)); + return 2; +} + +/* + * Branch conversion tests. Complex operations can expand to a lot + * of instructions when JITed. This in turn may cause jump offsets + * to overflow the field size of the native instruction, triggering + * a branch conversion mechanism in some JITs. + */ +static int __bpf_fill_max_jmp(struct bpf_test *self, int jmp, int imm) +{ + struct bpf_insn *insns; + int len = S16_MAX + 5; int i; - insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL); - if (!insn) + insns = kmalloc_array(len, sizeof(*insns), GFP_KERNEL); + if (!insns) return -ENOMEM; - insn[0] = BPF_ALU64_IMM(BPF_MOV, R0, 1); - insn[1] = BPF_JMP_IMM(BPF_JEQ, R0, 1, len - 2 - 1); + i = __bpf_ld_imm64(insns, R1, 0x0123456789abcdefULL); + insns[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); + insns[i++] = BPF_JMP_IMM(jmp, R0, imm, S16_MAX); + insns[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 2); + insns[i++] = BPF_EXIT_INSN(); - /* - * Fill with a complex 64-bit operation that expands to a lot of - * instructions on 32-bit JITs. The large jump offset can then - * overflow the conditional branch field size, triggering a branch - * conversion mechanism in some JITs. - * - * Note: BPF_MAXINSNS of ALU64 MUL is enough to trigger such branch - * conversion on the 32-bit MIPS JIT. For other JITs, the instruction - * count and/or operation may need to be modified to trigger the - * branch conversion. - */ - for (i = 2; i < len - 1; i++) - insn[i] = BPF_ALU64_IMM(BPF_MUL, R0, (i << 16) + i); + while (i < len - 1) { + static const int ops[] = { + BPF_LSH, BPF_RSH, BPF_ARSH, BPF_ADD, + BPF_SUB, BPF_MUL, BPF_DIV, BPF_MOD, + }; + int op = ops[(i >> 1) % ARRAY_SIZE(ops)]; - insn[len - 1] = BPF_EXIT_INSN(); + if (i & 1) + insns[i++] = BPF_ALU32_REG(op, R0, R1); + else + insns[i++] = BPF_ALU64_REG(op, R0, R1); + } - self->u.ptr.insns = insn; + insns[i++] = BPF_EXIT_INSN(); + self->u.ptr.insns = insns; self->u.ptr.len = len; + BUG_ON(i != len); return 0; } -static int __bpf_ld_imm64(struct bpf_insn insns[2], u8 reg, s64 imm64) +/* Branch taken by runtime decision */ +static int bpf_fill_max_jmp_taken(struct bpf_test *self) { - struct bpf_insn tmp[] = {BPF_LD_IMM64(reg, imm64)}; + return __bpf_fill_max_jmp(self, BPF_JEQ, 1); +} - memcpy(insns, tmp, sizeof(tmp)); - return 2; +/* Branch not taken by runtime decision */ +static int bpf_fill_max_jmp_not_taken(struct bpf_test *self) +{ + return __bpf_fill_max_jmp(self, BPF_JEQ, 0); +} + +/* Branch always taken, known at JIT time */ +static int bpf_fill_max_jmp_always_taken(struct bpf_test *self) +{ + return __bpf_fill_max_jmp(self, BPF_JGE, 0); +} + +/* Branch never taken, known at JIT time */ +static int bpf_fill_max_jmp_never_taken(struct bpf_test *self) +{ + return __bpf_fill_max_jmp(self, BPF_JLT, 0); } /* Test an ALU shift operation for all valid shift values */ @@ -8653,14 +8685,6 @@ static struct bpf_test tests[] = { { }, { { 0, 1 } }, }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Very long conditional jump", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_long_jmp, - }, { "JMP_JA: Jump, gap, jump, ...", { }, @@ -11009,6 +11033,39 @@ static struct bpf_test tests[] = { { }, { { 0, 0 } }, }, + /* Conditional branch conversions */ + { + "Long conditional jump: taken at runtime", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_max_jmp_taken, + }, + { + "Long conditional jump: not taken at runtime", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 2 } }, + .fill_helper = bpf_fill_max_jmp_not_taken, + }, + { + "Long conditional jump: always taken, known at JIT time", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_max_jmp_always_taken, + }, + { + "Long conditional jump: never taken, known at JIT time", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 2 } }, + .fill_helper = bpf_fill_max_jmp_never_taken, + }, /* Staggered jump sequences, immediate */ { "Staggered jumps: JMP_JA", -- cgit v1.2.3 From f536a7c80675e4875e50df9182881d7678e27651 Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Tue, 14 Sep 2021 11:18:40 +0200 Subject: bpf/tests: Add more BPF_END byte order conversion tests This patch adds tests of the high 32 bits of 64-bit BPF_END conversions. It also adds a mirrored set of tests where the source bytes are reversed. The MSB of each byte is now set on the high word instead, possibly affecting sign-extension during conversion in a different way. Mainly for JIT testing. Signed-off-by: Johan Almbladh Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20210914091842.4186267-13-johan.almbladh@anyfinetworks.com --- lib/test_bpf.c | 122 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index dd6bb0044097..7f8ad4fa4add 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -6748,6 +6748,67 @@ static struct bpf_test tests[] = { { }, { { 0, (u32) cpu_to_be64(0x0123456789abcdefLL) } }, }, + { + "ALU_END_FROM_BE 64: 0x0123456789abcdef >> 32 -> 0x01234567", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ENDIAN(BPF_FROM_BE, R0, 64), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, (u32) (cpu_to_be64(0x0123456789abcdefLL) >> 32) } }, + }, + /* BPF_ALU | BPF_END | BPF_FROM_BE, reversed */ + { + "ALU_END_FROM_BE 16: 0xfedcba9876543210 -> 0x3210", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_ENDIAN(BPF_FROM_BE, R0, 16), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, cpu_to_be16(0x3210) } }, + }, + { + "ALU_END_FROM_BE 32: 0xfedcba9876543210 -> 0x76543210", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_ENDIAN(BPF_FROM_BE, R0, 32), + BPF_ALU64_REG(BPF_MOV, R1, R0), + BPF_ALU64_IMM(BPF_RSH, R1, 32), + BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, cpu_to_be32(0x76543210) } }, + }, + { + "ALU_END_FROM_BE 64: 0xfedcba9876543210 -> 0x76543210", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_ENDIAN(BPF_FROM_BE, R0, 64), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, (u32) cpu_to_be64(0xfedcba9876543210ULL) } }, + }, + { + "ALU_END_FROM_BE 64: 0xfedcba9876543210 >> 32 -> 0xfedcba98", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_ENDIAN(BPF_FROM_BE, R0, 64), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, (u32) (cpu_to_be64(0xfedcba9876543210ULL) >> 32) } }, + }, /* BPF_ALU | BPF_END | BPF_FROM_LE */ { "ALU_END_FROM_LE 16: 0x0123456789abcdef -> 0xefcd", @@ -6785,6 +6846,67 @@ static struct bpf_test tests[] = { { }, { { 0, (u32) cpu_to_le64(0x0123456789abcdefLL) } }, }, + { + "ALU_END_FROM_LE 64: 0x0123456789abcdef >> 32 -> 0xefcdab89", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ENDIAN(BPF_FROM_LE, R0, 64), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, (u32) (cpu_to_le64(0x0123456789abcdefLL) >> 32) } }, + }, + /* BPF_ALU | BPF_END | BPF_FROM_LE, reversed */ + { + "ALU_END_FROM_LE 16: 0xfedcba9876543210 -> 0x1032", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_ENDIAN(BPF_FROM_LE, R0, 16), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, cpu_to_le16(0x3210) } }, + }, + { + "ALU_END_FROM_LE 32: 0xfedcba9876543210 -> 0x10325476", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_ENDIAN(BPF_FROM_LE, R0, 32), + BPF_ALU64_REG(BPF_MOV, R1, R0), + BPF_ALU64_IMM(BPF_RSH, R1, 32), + BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, cpu_to_le32(0x76543210) } }, + }, + { + "ALU_END_FROM_LE 64: 0xfedcba9876543210 -> 0x10325476", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_ENDIAN(BPF_FROM_LE, R0, 64), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, (u32) cpu_to_le64(0xfedcba9876543210ULL) } }, + }, + { + "ALU_END_FROM_LE 64: 0xfedcba9876543210 >> 32 -> 0x98badcfe", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_ENDIAN(BPF_FROM_LE, R0, 64), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, (u32) (cpu_to_le64(0xfedcba9876543210ULL) >> 32) } }, + }, /* BPF_ST(X) | BPF_MEM | BPF_B/H/W/DW */ { "ST_MEM_B: Store/Load byte: max negative", -- cgit v1.2.3 From 18935a72eb25525b655262579e1652362a3b29bb Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Tue, 14 Sep 2021 11:18:41 +0200 Subject: bpf/tests: Fix error in tail call limit tests This patch fixes an error in the tail call limit test that caused the test to fail on for x86-64 JIT. Previously, the register R0 was used to report the total number of tail calls made. However, after a tail call fall-through, the value of the R0 register is undefined. Now, all tail call error path tests instead use context state to store the count. Fixes: 874be05f525e ("bpf, tests: Add tail call test suite") Reported-by: Paul Chaignon Reported-by: Tiezhu Yang Signed-off-by: Johan Almbladh Signed-off-by: Daniel Borkmann Tested-by: Tiezhu Yang Link: https://lore.kernel.org/bpf/20210914091842.4186267-14-johan.almbladh@anyfinetworks.com --- lib/test_bpf.c | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index 7f8ad4fa4add..a94ab634f947 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -12180,10 +12180,15 @@ static __init int test_bpf(void) struct tail_call_test { const char *descr; struct bpf_insn insns[MAX_INSNS]; + int flags; int result; int stack_depth; }; +/* Flags that can be passed to tail call test cases */ +#define FLAG_NEED_STATE BIT(0) +#define FLAG_RESULT_IN_STATE BIT(1) + /* * Magic marker used in test snippets for tail calls below. * BPF_LD/MOV to R2 and R2 with this immediate value is replaced @@ -12253,32 +12258,38 @@ static struct tail_call_test tail_call_tests[] = { { "Tail call error path, max count reached", .insns = { - BPF_ALU64_IMM(BPF_ADD, R1, 1), - BPF_ALU64_REG(BPF_MOV, R0, R1), + BPF_LDX_MEM(BPF_W, R2, R1, 0), + BPF_ALU64_IMM(BPF_ADD, R2, 1), + BPF_STX_MEM(BPF_W, R1, R2, 0), TAIL_CALL(0), BPF_EXIT_INSN(), }, - .result = MAX_TAIL_CALL_CNT + 1, + .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, + .result = (MAX_TAIL_CALL_CNT + 1 + 1) * MAX_TESTRUNS, }, { "Tail call error path, NULL target", .insns = { - BPF_ALU64_IMM(BPF_MOV, R0, -1), + BPF_LDX_MEM(BPF_W, R2, R1, 0), + BPF_ALU64_IMM(BPF_ADD, R2, 1), + BPF_STX_MEM(BPF_W, R1, R2, 0), TAIL_CALL(TAIL_CALL_NULL), - BPF_ALU64_IMM(BPF_MOV, R0, 1), BPF_EXIT_INSN(), }, - .result = 1, + .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, + .result = MAX_TESTRUNS, }, { "Tail call error path, index out of range", .insns = { - BPF_ALU64_IMM(BPF_MOV, R0, -1), + BPF_LDX_MEM(BPF_W, R2, R1, 0), + BPF_ALU64_IMM(BPF_ADD, R2, 1), + BPF_STX_MEM(BPF_W, R1, R2, 0), TAIL_CALL(TAIL_CALL_INVALID), - BPF_ALU64_IMM(BPF_MOV, R0, 1), BPF_EXIT_INSN(), }, - .result = 1, + .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, + .result = MAX_TESTRUNS, }, }; @@ -12384,6 +12395,8 @@ static __init int test_tail_calls(struct bpf_array *progs) for (i = 0; i < ARRAY_SIZE(tail_call_tests); i++) { struct tail_call_test *test = &tail_call_tests[i]; struct bpf_prog *fp = progs->ptrs[i]; + int *data = NULL; + int state = 0; u64 duration; int ret; @@ -12400,7 +12413,11 @@ static __init int test_tail_calls(struct bpf_array *progs) if (fp->jited) jit_cnt++; - ret = __run_one(fp, NULL, MAX_TESTRUNS, &duration); + if (test->flags & FLAG_NEED_STATE) + data = &state; + ret = __run_one(fp, data, MAX_TESTRUNS, &duration); + if (test->flags & FLAG_RESULT_IN_STATE) + ret = state; if (ret == test->result) { pr_cont("%lld PASS", duration); pass_cnt++; -- cgit v1.2.3 From 29eef85be2f60b1027214b4bfc4b1a9d592830a7 Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Tue, 14 Sep 2021 11:18:42 +0200 Subject: bpf/tests: Add tail call limit test with external function call This patch adds a tail call limit test where the program also emits a BPF_CALL to an external function prior to the tail call. Mainly testing that JITed programs preserve its internal register state, for example tail call count, across such external calls. Signed-off-by: Johan Almbladh Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20210914091842.4186267-15-johan.almbladh@anyfinetworks.com --- lib/test_bpf.c | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 83 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index a94ab634f947..08f438e6fe9e 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -12208,6 +12208,30 @@ struct tail_call_test { offset, TAIL_CALL_MARKER), \ BPF_JMP_IMM(BPF_TAIL_CALL, 0, 0, 0) +/* + * A test function to be called from a BPF program, clobbering a lot of + * CPU registers in the process. A JITed BPF program calling this function + * must save and restore any caller-saved registers it uses for internal + * state, for example the current tail call count. + */ +BPF_CALL_1(bpf_test_func, u64, arg) +{ + char buf[64]; + long a = 0; + long b = 1; + long c = 2; + long d = 3; + long e = 4; + long f = 5; + long g = 6; + long h = 7; + + return snprintf(buf, sizeof(buf), + "%ld %lu %lx %ld %lu %lx %ld %lu %x", + a, b, c, d, e, f, g, h, (int)arg); +} +#define BPF_FUNC_test_func __BPF_FUNC_MAX_ID + /* * Tail call tests. Each test case may call any other test in the table, * including itself, specified as a relative index offset from the calling @@ -12267,6 +12291,28 @@ static struct tail_call_test tail_call_tests[] = { .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, .result = (MAX_TAIL_CALL_CNT + 1 + 1) * MAX_TESTRUNS, }, + { + "Tail call count preserved across function calls", + .insns = { + BPF_LDX_MEM(BPF_W, R2, R1, 0), + BPF_ALU64_IMM(BPF_ADD, R2, 1), + BPF_STX_MEM(BPF_W, R1, R2, 0), + BPF_STX_MEM(BPF_DW, R10, R1, -8), + BPF_CALL_REL(BPF_FUNC_get_numa_node_id), + BPF_CALL_REL(BPF_FUNC_ktime_get_ns), + BPF_CALL_REL(BPF_FUNC_ktime_get_boot_ns), + BPF_CALL_REL(BPF_FUNC_ktime_get_coarse_ns), + BPF_CALL_REL(BPF_FUNC_jiffies64), + BPF_CALL_REL(BPF_FUNC_test_func), + BPF_LDX_MEM(BPF_DW, R1, R10, -8), + BPF_ALU32_REG(BPF_MOV, R0, R1), + TAIL_CALL(0), + BPF_EXIT_INSN(), + }, + .stack_depth = 8, + .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, + .result = (MAX_TAIL_CALL_CNT + 1 + 1) * MAX_TESTRUNS, + }, { "Tail call error path, NULL target", .insns = { @@ -12345,17 +12391,19 @@ static __init int prepare_tail_call_tests(struct bpf_array **pprogs) /* Relocate runtime tail call offsets and addresses */ for (i = 0; i < len; i++) { struct bpf_insn *insn = &fp->insnsi[i]; - - if (insn->imm != TAIL_CALL_MARKER) - continue; + long addr = 0; switch (insn->code) { case BPF_LD | BPF_DW | BPF_IMM: + if (insn->imm != TAIL_CALL_MARKER) + break; insn[0].imm = (u32)(long)progs; insn[1].imm = ((u64)(long)progs) >> 32; break; case BPF_ALU | BPF_MOV | BPF_K: + if (insn->imm != TAIL_CALL_MARKER) + break; if (insn->off == TAIL_CALL_NULL) insn->imm = ntests; else if (insn->off == TAIL_CALL_INVALID) @@ -12363,6 +12411,38 @@ static __init int prepare_tail_call_tests(struct bpf_array **pprogs) else insn->imm = which + insn->off; insn->off = 0; + break; + + case BPF_JMP | BPF_CALL: + if (insn->src_reg != BPF_PSEUDO_CALL) + break; + switch (insn->imm) { + case BPF_FUNC_get_numa_node_id: + addr = (long)&numa_node_id; + break; + case BPF_FUNC_ktime_get_ns: + addr = (long)&ktime_get_ns; + break; + case BPF_FUNC_ktime_get_boot_ns: + addr = (long)&ktime_get_boot_fast_ns; + break; + case BPF_FUNC_ktime_get_coarse_ns: + addr = (long)&ktime_get_coarse_ns; + break; + case BPF_FUNC_jiffies64: + addr = (long)&get_jiffies_64; + break; + case BPF_FUNC_test_func: + addr = (long)&bpf_test_func; + break; + default: + err = -EFAULT; + goto out_err; + } + *insn = BPF_EMIT_CALL(BPF_CAST_CALL(addr)); + if ((long)__bpf_call_base + insn->imm != addr) + *insn = BPF_JMP_A(0); /* Skip: NOP */ + break; } } -- cgit v1.2.3 From 3d717fad5081b8e3bda76d86907fad95398cbde8 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Tue, 28 Sep 2021 16:09:45 -0700 Subject: bpf: Replace "want address" users of BPF_CAST_CALL with BPF_CALL_IMM In order to keep ahead of cases in the kernel where Control Flow Integrity (CFI) may trip over function call casts, enabling -Wcast-function-type is helpful. To that end, BPF_CAST_CALL causes various warnings and is one of the last places in the kernel triggering this warning. Most places using BPF_CAST_CALL actually just want a void * to perform math on. It's not actually performing a call, so just use a different helper to get the void *, by way of the new BPF_CALL_IMM() helper, which can clean up a common copy/paste idiom as well. This change results in no object code difference. Signed-off-by: Kees Cook Signed-off-by: Alexei Starovoitov Reviewed-by: Gustavo A. R. Silva Acked-by: Andrii Nakryiko Link: https://github.com/KSPP/linux/issues/20 Link: https://lore.kernel.org/lkml/CAEf4Bzb46=-J5Fxc3mMZ8JQPtK1uoE0q6+g6WPz53Cvx=CBEhw@mail.gmail.com Link: https://lore.kernel.org/bpf/20210928230946.4062144-2-keescook@chromium.org --- include/linux/filter.h | 6 +++++- kernel/bpf/hashtab.c | 6 +++--- kernel/bpf/verifier.c | 26 +++++++++----------------- lib/test_bpf.c | 2 +- 4 files changed, 18 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/include/linux/filter.h b/include/linux/filter.h index 4a93c12543ee..6c247663d4ce 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -365,13 +365,17 @@ static inline bool insn_is_zext(const struct bpf_insn *insn) #define BPF_CAST_CALL(x) \ ((u64 (*)(u64, u64, u64, u64, u64))(x)) +/* Convert function address to BPF immediate */ + +#define BPF_CALL_IMM(x) ((void *)(x) - (void *)__bpf_call_base) + #define BPF_EMIT_CALL(FUNC) \ ((struct bpf_insn) { \ .code = BPF_JMP | BPF_CALL, \ .dst_reg = 0, \ .src_reg = 0, \ .off = 0, \ - .imm = ((FUNC) - __bpf_call_base) }) + .imm = BPF_CALL_IMM(FUNC) }) /* Raw code statement block */ diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 32471ba02708..3d8f9d6997d5 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -668,7 +668,7 @@ static int htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem, (void *(*)(struct bpf_map *map, void *key))NULL)); - *insn++ = BPF_EMIT_CALL(BPF_CAST_CALL(__htab_map_lookup_elem)); + *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1); *insn++ = BPF_ALU64_IMM(BPF_ADD, ret, offsetof(struct htab_elem, key) + @@ -709,7 +709,7 @@ static int htab_lru_map_gen_lookup(struct bpf_map *map, BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem, (void *(*)(struct bpf_map *map, void *key))NULL)); - *insn++ = BPF_EMIT_CALL(BPF_CAST_CALL(__htab_map_lookup_elem)); + *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 4); *insn++ = BPF_LDX_MEM(BPF_B, ref_reg, ret, offsetof(struct htab_elem, lru_node) + @@ -2397,7 +2397,7 @@ static int htab_of_map_gen_lookup(struct bpf_map *map, BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem, (void *(*)(struct bpf_map *map, void *key))NULL)); - *insn++ = BPF_EMIT_CALL(BPF_CAST_CALL(__htab_map_lookup_elem)); + *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 2); *insn++ = BPF_ALU64_IMM(BPF_ADD, ret, offsetof(struct htab_elem, key) + diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 7a8351604f67..1433752db740 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -1744,7 +1744,7 @@ static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id) desc = &tab->descs[tab->nr_descs++]; desc->func_id = func_id; - desc->imm = BPF_CAST_CALL(addr) - __bpf_call_base; + desc->imm = BPF_CALL_IMM(addr); err = btf_distill_func_proto(&env->log, btf_vmlinux, func_proto, func_name, &desc->func_model); @@ -12514,8 +12514,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) if (!bpf_pseudo_call(insn)) continue; subprog = insn->off; - insn->imm = BPF_CAST_CALL(func[subprog]->bpf_func) - - __bpf_call_base; + insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func); } /* we use the aux data to keep a list of the start addresses @@ -12995,32 +12994,25 @@ static int do_misc_fixups(struct bpf_verifier_env *env) patch_map_ops_generic: switch (insn->imm) { case BPF_FUNC_map_lookup_elem: - insn->imm = BPF_CAST_CALL(ops->map_lookup_elem) - - __bpf_call_base; + insn->imm = BPF_CALL_IMM(ops->map_lookup_elem); continue; case BPF_FUNC_map_update_elem: - insn->imm = BPF_CAST_CALL(ops->map_update_elem) - - __bpf_call_base; + insn->imm = BPF_CALL_IMM(ops->map_update_elem); continue; case BPF_FUNC_map_delete_elem: - insn->imm = BPF_CAST_CALL(ops->map_delete_elem) - - __bpf_call_base; + insn->imm = BPF_CALL_IMM(ops->map_delete_elem); continue; case BPF_FUNC_map_push_elem: - insn->imm = BPF_CAST_CALL(ops->map_push_elem) - - __bpf_call_base; + insn->imm = BPF_CALL_IMM(ops->map_push_elem); continue; case BPF_FUNC_map_pop_elem: - insn->imm = BPF_CAST_CALL(ops->map_pop_elem) - - __bpf_call_base; + insn->imm = BPF_CALL_IMM(ops->map_pop_elem); continue; case BPF_FUNC_map_peek_elem: - insn->imm = BPF_CAST_CALL(ops->map_peek_elem) - - __bpf_call_base; + insn->imm = BPF_CALL_IMM(ops->map_peek_elem); continue; case BPF_FUNC_redirect_map: - insn->imm = BPF_CAST_CALL(ops->map_redirect) - - __bpf_call_base; + insn->imm = BPF_CALL_IMM(ops->map_redirect); continue; } diff --git a/lib/test_bpf.c b/lib/test_bpf.c index 08f438e6fe9e..21ea1ab253a1 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -12439,7 +12439,7 @@ static __init int prepare_tail_call_tests(struct bpf_array **pprogs) err = -EFAULT; goto out_err; } - *insn = BPF_EMIT_CALL(BPF_CAST_CALL(addr)); + *insn = BPF_EMIT_CALL(addr); if ((long)__bpf_call_base + insn->imm != addr) *insn = BPF_JMP_A(0); /* Skip: NOP */ break; -- cgit v1.2.3 From caaaa1667bf198c54cc3141ad92ca6ce853e99cd Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Fri, 1 Oct 2021 15:03:39 +0200 Subject: bpf/tests: Add tests of BPF_LDX and BPF_STX with small sizes This patch adds a series of tests to verify the behavior of BPF_LDX and BPF_STX with BPF_B//W sizes in isolation. In particular, it checks that BPF_LDX zero-extendeds the result, and that BPF_STX does not overwrite adjacent bytes in memory. BPF_ST and operations on BPF_DW size are deemed to be sufficiently tested by existing tests. Signed-off-by: Johan Almbladh Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20211001130348.3670534-2-johan.almbladh@anyfinetworks.com --- lib/test_bpf.c | 254 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 254 insertions(+) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index 21ea1ab253a1..a838a6179ca4 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -6907,6 +6907,260 @@ static struct bpf_test tests[] = { { }, { { 0, (u32) (cpu_to_le64(0xfedcba9876543210ULL) >> 32) } }, }, + /* BPF_LDX_MEM B/H/W/DW */ + { + "BPF_LDX_MEM | BPF_B", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x0102030405060708ULL), + BPF_LD_IMM64(R2, 0x0000000000000008ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_LDX_MEM(BPF_B, R0, R10, -1), +#else + BPF_LDX_MEM(BPF_B, R0, R10, -8), +#endif + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_LDX_MEM | BPF_B, MSB set", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8182838485868788ULL), + BPF_LD_IMM64(R2, 0x0000000000000088ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_LDX_MEM(BPF_B, R0, R10, -1), +#else + BPF_LDX_MEM(BPF_B, R0, R10, -8), +#endif + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_LDX_MEM | BPF_H", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x0102030405060708ULL), + BPF_LD_IMM64(R2, 0x0000000000000708ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_LDX_MEM(BPF_H, R0, R10, -2), +#else + BPF_LDX_MEM(BPF_H, R0, R10, -8), +#endif + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_LDX_MEM | BPF_H, MSB set", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8182838485868788ULL), + BPF_LD_IMM64(R2, 0x0000000000008788ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_LDX_MEM(BPF_H, R0, R10, -2), +#else + BPF_LDX_MEM(BPF_H, R0, R10, -8), +#endif + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_LDX_MEM | BPF_W", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x0102030405060708ULL), + BPF_LD_IMM64(R2, 0x0000000005060708ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_LDX_MEM(BPF_W, R0, R10, -4), +#else + BPF_LDX_MEM(BPF_W, R0, R10, -8), +#endif + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_LDX_MEM | BPF_W, MSB set", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8182838485868788ULL), + BPF_LD_IMM64(R2, 0x0000000085868788ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_LDX_MEM(BPF_W, R0, R10, -4), +#else + BPF_LDX_MEM(BPF_W, R0, R10, -8), +#endif + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + /* BPF_STX_MEM B/H/W/DW */ + { + "BPF_STX_MEM | BPF_B", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), + BPF_LD_IMM64(R2, 0x0102030405060708ULL), + BPF_LD_IMM64(R3, 0x8090a0b0c0d0e008ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_STX_MEM(BPF_B, R10, R2, -1), +#else + BPF_STX_MEM(BPF_B, R10, R2, -8), +#endif + BPF_LDX_MEM(BPF_DW, R0, R10, -8), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_STX_MEM | BPF_B, MSB set", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), + BPF_LD_IMM64(R2, 0x8182838485868788ULL), + BPF_LD_IMM64(R3, 0x8090a0b0c0d0e088ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_STX_MEM(BPF_B, R10, R2, -1), +#else + BPF_STX_MEM(BPF_B, R10, R2, -8), +#endif + BPF_LDX_MEM(BPF_DW, R0, R10, -8), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_STX_MEM | BPF_H", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), + BPF_LD_IMM64(R2, 0x0102030405060708ULL), + BPF_LD_IMM64(R3, 0x8090a0b0c0d00708ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_STX_MEM(BPF_H, R10, R2, -2), +#else + BPF_STX_MEM(BPF_H, R10, R2, -8), +#endif + BPF_LDX_MEM(BPF_DW, R0, R10, -8), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_STX_MEM | BPF_H, MSB set", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), + BPF_LD_IMM64(R2, 0x8182838485868788ULL), + BPF_LD_IMM64(R3, 0x8090a0b0c0d08788ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_STX_MEM(BPF_H, R10, R2, -2), +#else + BPF_STX_MEM(BPF_H, R10, R2, -8), +#endif + BPF_LDX_MEM(BPF_DW, R0, R10, -8), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_STX_MEM | BPF_W", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), + BPF_LD_IMM64(R2, 0x0102030405060708ULL), + BPF_LD_IMM64(R3, 0x8090a0b005060708ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_STX_MEM(BPF_W, R10, R2, -4), +#else + BPF_STX_MEM(BPF_W, R10, R2, -8), +#endif + BPF_LDX_MEM(BPF_DW, R0, R10, -8), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_STX_MEM | BPF_W, MSB set", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), + BPF_LD_IMM64(R2, 0x8182838485868788ULL), + BPF_LD_IMM64(R3, 0x8090a0b085868788ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_STX_MEM(BPF_W, R10, R2, -4), +#else + BPF_STX_MEM(BPF_W, R10, R2, -8), +#endif + BPF_LDX_MEM(BPF_DW, R0, R10, -8), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, /* BPF_ST(X) | BPF_MEM | BPF_B/H/W/DW */ { "ST_MEM_B: Store/Load byte: max negative", -- cgit v1.2.3 From 89b63462765cc0370f22ebec53d3e83cbbb17613 Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Fri, 1 Oct 2021 15:03:40 +0200 Subject: bpf/tests: Add zero-extension checks in BPF_ATOMIC tests This patch updates the existing tests of BPF_ATOMIC operations to verify that a 32-bit register operand is properly zero-extended. In particular, it checks the operation on archs that require 32-bit operands to be properly zero-/sign-extended or the result is undefined, e.g. MIPS64. Signed-off-by: Johan Almbladh Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20211001130348.3670534-3-johan.almbladh@anyfinetworks.com --- lib/test_bpf.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index a838a6179ca4..f6983ad7b981 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -7398,15 +7398,20 @@ static struct bpf_test tests[] = { * Individual tests are expanded from template macros for all * combinations of ALU operation, word size and fetching. */ +#define BPF_ATOMIC_POISON(width) ((width) == BPF_W ? (0xbaadf00dULL << 32) : 0) + #define BPF_ATOMIC_OP_TEST1(width, op, logic, old, update, result) \ { \ "BPF_ATOMIC | " #width ", " #op ": Test: " \ #old " " #logic " " #update " = " #result, \ .u.insns_int = { \ - BPF_ALU32_IMM(BPF_MOV, R5, update), \ + BPF_LD_IMM64(R5, (update) | BPF_ATOMIC_POISON(width)), \ BPF_ST_MEM(width, R10, -40, old), \ BPF_ATOMIC_OP(width, op, R10, R5, -40), \ BPF_LDX_MEM(width, R0, R10, -40), \ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_ALU64_IMM(BPF_RSH, R1, 32), \ + BPF_ALU64_REG(BPF_OR, R0, R1), \ BPF_EXIT_INSN(), \ }, \ INTERNAL, \ @@ -7420,11 +7425,14 @@ static struct bpf_test tests[] = { #old " " #logic " " #update " = " #result, \ .u.insns_int = { \ BPF_ALU64_REG(BPF_MOV, R1, R10), \ - BPF_ALU32_IMM(BPF_MOV, R0, update), \ + BPF_LD_IMM64(R0, (update) | BPF_ATOMIC_POISON(width)), \ BPF_ST_MEM(BPF_W, R10, -40, old), \ BPF_ATOMIC_OP(width, op, R10, R0, -40), \ BPF_ALU64_REG(BPF_MOV, R0, R10), \ BPF_ALU64_REG(BPF_SUB, R0, R1), \ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_ALU64_IMM(BPF_RSH, R1, 32), \ + BPF_ALU64_REG(BPF_OR, R0, R1), \ BPF_EXIT_INSN(), \ }, \ INTERNAL, \ @@ -7438,10 +7446,13 @@ static struct bpf_test tests[] = { #old " " #logic " " #update " = " #result, \ .u.insns_int = { \ BPF_ALU64_REG(BPF_MOV, R0, R10), \ - BPF_ALU32_IMM(BPF_MOV, R1, update), \ + BPF_LD_IMM64(R1, (update) | BPF_ATOMIC_POISON(width)), \ BPF_ST_MEM(width, R10, -40, old), \ BPF_ATOMIC_OP(width, op, R10, R1, -40), \ BPF_ALU64_REG(BPF_SUB, R0, R10), \ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_ALU64_IMM(BPF_RSH, R1, 32), \ + BPF_ALU64_REG(BPF_OR, R0, R1), \ BPF_EXIT_INSN(), \ }, \ INTERNAL, \ @@ -7454,10 +7465,10 @@ static struct bpf_test tests[] = { "BPF_ATOMIC | " #width ", " #op ": Test fetch: " \ #old " " #logic " " #update " = " #result, \ .u.insns_int = { \ - BPF_ALU32_IMM(BPF_MOV, R3, update), \ + BPF_LD_IMM64(R3, (update) | BPF_ATOMIC_POISON(width)), \ BPF_ST_MEM(width, R10, -40, old), \ BPF_ATOMIC_OP(width, op, R10, R3, -40), \ - BPF_ALU64_REG(BPF_MOV, R0, R3), \ + BPF_ALU32_REG(BPF_MOV, R0, R3), \ BPF_EXIT_INSN(), \ }, \ INTERNAL, \ @@ -7555,6 +7566,7 @@ static struct bpf_test tests[] = { BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab), BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab), BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab), +#undef BPF_ATOMIC_POISON #undef BPF_ATOMIC_OP_TEST1 #undef BPF_ATOMIC_OP_TEST2 #undef BPF_ATOMIC_OP_TEST3 -- cgit v1.2.3 From f68e8efd7fa506928432b8cd41b8c7d91d804e02 Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Fri, 1 Oct 2021 15:03:41 +0200 Subject: bpf/tests: Add exhaustive tests of BPF_ATOMIC magnitudes This patch adds a series of test to verify the operation of BPF_ATOMIC with BPF_DW and BPF_W sizes, for all power-of-two magnitudes of the register value operand. Also fixes a confusing typo in the comment for a related test. Signed-off-by: Johan Almbladh Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20211001130348.3670534-4-johan.almbladh@anyfinetworks.com --- lib/test_bpf.c | 504 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 503 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index f6983ad7b981..84efb23e09d0 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -796,7 +796,7 @@ static int __bpf_fill_pattern(struct bpf_test *self, void *arg, /* * Exhaustive tests of ALU operations for all combinations of power-of-two * magnitudes of the operands, both for positive and negative values. The - * test is designed to verify e.g. the JMP and JMP32 operations for JITs that + * test is designed to verify e.g. the ALU and ALU64 operations for JITs that * emit different code depending on the magnitude of the immediate value. */ @@ -1137,6 +1137,306 @@ static int bpf_fill_alu32_mod_reg(struct bpf_test *self) return __bpf_fill_alu32_reg(self, BPF_MOD); } +/* + * Exhaustive tests of atomic operations for all power-of-two operand + * magnitudes, both for positive and negative values. + */ + +static int __bpf_emit_atomic64(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 src) +{ + int op = *(int *)arg; + u64 keep, fetch, res; + int i = 0; + + if (!insns) + return 21; + + switch (op) { + case BPF_XCHG: + res = src; + break; + default: + __bpf_alu_result(&res, dst, src, BPF_OP(op)); + } + + keep = 0x0123456789abcdefULL; + if (op & BPF_FETCH) + fetch = dst; + else + fetch = src; + + i += __bpf_ld_imm64(&insns[i], R0, keep); + i += __bpf_ld_imm64(&insns[i], R1, dst); + i += __bpf_ld_imm64(&insns[i], R2, src); + i += __bpf_ld_imm64(&insns[i], R3, res); + i += __bpf_ld_imm64(&insns[i], R4, fetch); + i += __bpf_ld_imm64(&insns[i], R5, keep); + + insns[i++] = BPF_STX_MEM(BPF_DW, R10, R1, -8); + insns[i++] = BPF_ATOMIC_OP(BPF_DW, op, R10, R2, -8); + insns[i++] = BPF_LDX_MEM(BPF_DW, R1, R10, -8); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R2, R4, 1); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R5, 1); + insns[i++] = BPF_EXIT_INSN(); + + return i; +} + +static int __bpf_emit_atomic32(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 src) +{ + int op = *(int *)arg; + u64 keep, fetch, res; + int i = 0; + + if (!insns) + return 21; + + switch (op) { + case BPF_XCHG: + res = src; + break; + default: + __bpf_alu_result(&res, (u32)dst, (u32)src, BPF_OP(op)); + } + + keep = 0x0123456789abcdefULL; + if (op & BPF_FETCH) + fetch = (u32)dst; + else + fetch = src; + + i += __bpf_ld_imm64(&insns[i], R0, keep); + i += __bpf_ld_imm64(&insns[i], R1, (u32)dst); + i += __bpf_ld_imm64(&insns[i], R2, src); + i += __bpf_ld_imm64(&insns[i], R3, (u32)res); + i += __bpf_ld_imm64(&insns[i], R4, fetch); + i += __bpf_ld_imm64(&insns[i], R5, keep); + + insns[i++] = BPF_STX_MEM(BPF_W, R10, R1, -4); + insns[i++] = BPF_ATOMIC_OP(BPF_W, op, R10, R2, -4); + insns[i++] = BPF_LDX_MEM(BPF_W, R1, R10, -4); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R2, R4, 1); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R5, 1); + insns[i++] = BPF_EXIT_INSN(); + + return i; +} + +static int __bpf_emit_cmpxchg64(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 src) +{ + int i = 0; + + if (!insns) + return 23; + + i += __bpf_ld_imm64(&insns[i], R0, ~dst); + i += __bpf_ld_imm64(&insns[i], R1, dst); + i += __bpf_ld_imm64(&insns[i], R2, src); + + /* Result unsuccessful */ + insns[i++] = BPF_STX_MEM(BPF_DW, R10, R1, -8); + insns[i++] = BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -8); + insns[i++] = BPF_LDX_MEM(BPF_DW, R3, R10, -8); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 2); + insns[i++] = BPF_MOV64_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R3, 2); + insns[i++] = BPF_MOV64_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + /* Result successful */ + insns[i++] = BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -8); + insns[i++] = BPF_LDX_MEM(BPF_DW, R3, R10, -8); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R2, R3, 2); + insns[i++] = BPF_MOV64_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R1, 2); + insns[i++] = BPF_MOV64_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + return i; +} + +static int __bpf_emit_cmpxchg32(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 src) +{ + int i = 0; + + if (!insns) + return 27; + + i += __bpf_ld_imm64(&insns[i], R0, ~dst); + i += __bpf_ld_imm64(&insns[i], R1, (u32)dst); + i += __bpf_ld_imm64(&insns[i], R2, src); + + /* Result unsuccessful */ + insns[i++] = BPF_STX_MEM(BPF_W, R10, R1, -4); + insns[i++] = BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R2, -4); + insns[i++] = BPF_ZEXT_REG(R0), /* Zext always inserted by verifier */ + insns[i++] = BPF_LDX_MEM(BPF_W, R3, R10, -4); + + insns[i++] = BPF_JMP32_REG(BPF_JEQ, R1, R3, 2); + insns[i++] = BPF_MOV32_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R3, 2); + insns[i++] = BPF_MOV32_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + /* Result successful */ + i += __bpf_ld_imm64(&insns[i], R0, dst); + insns[i++] = BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R2, -4); + insns[i++] = BPF_ZEXT_REG(R0), /* Zext always inserted by verifier */ + insns[i++] = BPF_LDX_MEM(BPF_W, R3, R10, -4); + + insns[i++] = BPF_JMP32_REG(BPF_JEQ, R2, R3, 2); + insns[i++] = BPF_MOV32_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R1, 2); + insns[i++] = BPF_MOV32_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + return i; +} + +static int __bpf_fill_atomic64(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 64, + 0, PATTERN_BLOCK2, + &__bpf_emit_atomic64); +} + +static int __bpf_fill_atomic32(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 64, + 0, PATTERN_BLOCK2, + &__bpf_emit_atomic32); +} + +/* 64-bit atomic operations */ +static int bpf_fill_atomic64_add(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_ADD); +} + +static int bpf_fill_atomic64_and(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_AND); +} + +static int bpf_fill_atomic64_or(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_OR); +} + +static int bpf_fill_atomic64_xor(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_XOR); +} + +static int bpf_fill_atomic64_add_fetch(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_ADD | BPF_FETCH); +} + +static int bpf_fill_atomic64_and_fetch(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_AND | BPF_FETCH); +} + +static int bpf_fill_atomic64_or_fetch(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_OR | BPF_FETCH); +} + +static int bpf_fill_atomic64_xor_fetch(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_XOR | BPF_FETCH); +} + +static int bpf_fill_atomic64_xchg(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_XCHG); +} + +static int bpf_fill_cmpxchg64(struct bpf_test *self) +{ + return __bpf_fill_pattern(self, NULL, 64, 64, 0, PATTERN_BLOCK2, + &__bpf_emit_cmpxchg64); +} + +/* 32-bit atomic operations */ +static int bpf_fill_atomic32_add(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_ADD); +} + +static int bpf_fill_atomic32_and(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_AND); +} + +static int bpf_fill_atomic32_or(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_OR); +} + +static int bpf_fill_atomic32_xor(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_XOR); +} + +static int bpf_fill_atomic32_add_fetch(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_ADD | BPF_FETCH); +} + +static int bpf_fill_atomic32_and_fetch(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_AND | BPF_FETCH); +} + +static int bpf_fill_atomic32_or_fetch(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_OR | BPF_FETCH); +} + +static int bpf_fill_atomic32_xor_fetch(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_XOR | BPF_FETCH); +} + +static int bpf_fill_atomic32_xchg(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_XCHG); +} + +static int bpf_fill_cmpxchg32(struct bpf_test *self) +{ + return __bpf_fill_pattern(self, NULL, 64, 64, 0, PATTERN_BLOCK2, + &__bpf_emit_cmpxchg32); +} + /* * Test the two-instruction 64-bit immediate load operation for all * power-of-two magnitudes of the immediate operand. For each MSB, a block @@ -10721,6 +11021,208 @@ static struct bpf_test tests[] = { { { 0, 1 } }, .fill_helper = bpf_fill_ld_imm64, }, + /* 64-bit ATOMIC magnitudes */ + { + "ATOMIC_DW_ADD: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_add, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_AND: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_and, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_OR: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_or, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_XOR: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_xor, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_ADD_FETCH: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_add_fetch, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_AND_FETCH: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_and_fetch, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_OR_FETCH: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_or_fetch, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_XOR_FETCH: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_xor_fetch, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_XCHG: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_xchg, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_CMPXCHG: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_cmpxchg64, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* 64-bit atomic magnitudes */ + { + "ATOMIC_W_ADD: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_add, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_AND: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_and, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_OR: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_or, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_XOR: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_xor, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_ADD_FETCH: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_add_fetch, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_AND_FETCH: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_and_fetch, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_OR_FETCH: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_or_fetch, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_XOR_FETCH: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_xor_fetch, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_XCHG: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_xchg, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_CMPXCHG: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_cmpxchg32, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, /* JMP immediate magnitudes */ { "JMP_JSET_K: all immediate value magnitudes", -- cgit v1.2.3 From 0bbaa02b481682004cf812dbeca68272752a5e8a Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Fri, 1 Oct 2021 15:03:42 +0200 Subject: bpf/tests: Add tests to check source register zero-extension This patch adds tests to check that the source register is preserved when zero-extending a 32-bit value. In particular, it checks that the source operand is not zero-extended in-place. Signed-off-by: Johan Almbladh Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20211001130348.3670534-5-johan.almbladh@anyfinetworks.com --- lib/test_bpf.c | 143 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index 84efb23e09d0..c7db90112ef0 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -10586,6 +10586,149 @@ static struct bpf_test tests[] = { {}, { { 0, 2 } }, }, + /* Checking that ALU32 src is not zero extended in place */ +#define BPF_ALU32_SRC_ZEXT(op) \ + { \ + "ALU32_" #op "_X: src preserved in zext", \ + .u.insns_int = { \ + BPF_LD_IMM64(R1, 0x0123456789acbdefULL),\ + BPF_LD_IMM64(R2, 0xfedcba9876543210ULL),\ + BPF_ALU64_REG(BPF_MOV, R0, R1), \ + BPF_ALU32_REG(BPF_##op, R2, R1), \ + BPF_ALU64_REG(BPF_SUB, R0, R1), \ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_ALU64_IMM(BPF_RSH, R1, 32), \ + BPF_ALU64_REG(BPF_OR, R0, R1), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, 0 } }, \ + } + BPF_ALU32_SRC_ZEXT(MOV), + BPF_ALU32_SRC_ZEXT(AND), + BPF_ALU32_SRC_ZEXT(OR), + BPF_ALU32_SRC_ZEXT(XOR), + BPF_ALU32_SRC_ZEXT(ADD), + BPF_ALU32_SRC_ZEXT(SUB), + BPF_ALU32_SRC_ZEXT(MUL), + BPF_ALU32_SRC_ZEXT(DIV), + BPF_ALU32_SRC_ZEXT(MOD), +#undef BPF_ALU32_SRC_ZEXT + /* Checking that ATOMIC32 src is not zero extended in place */ +#define BPF_ATOMIC32_SRC_ZEXT(op) \ + { \ + "ATOMIC_W_" #op ": src preserved in zext", \ + .u.insns_int = { \ + BPF_LD_IMM64(R0, 0x0123456789acbdefULL), \ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_ST_MEM(BPF_W, R10, -4, 0), \ + BPF_ATOMIC_OP(BPF_W, BPF_##op, R10, R1, -4), \ + BPF_ALU64_REG(BPF_SUB, R0, R1), \ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_ALU64_IMM(BPF_RSH, R1, 32), \ + BPF_ALU64_REG(BPF_OR, R0, R1), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, 0 } }, \ + .stack_depth = 8, \ + } + BPF_ATOMIC32_SRC_ZEXT(ADD), + BPF_ATOMIC32_SRC_ZEXT(AND), + BPF_ATOMIC32_SRC_ZEXT(OR), + BPF_ATOMIC32_SRC_ZEXT(XOR), +#undef BPF_ATOMIC32_SRC_ZEXT + /* Checking that CMPXCHG32 src is not zero extended in place */ + { + "ATOMIC_W_CMPXCHG: src preserved in zext", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x0123456789acbdefULL), + BPF_ALU64_REG(BPF_MOV, R2, R1), + BPF_ALU64_REG(BPF_MOV, R0, 0), + BPF_ST_MEM(BPF_W, R10, -4, 0), + BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R1, -4), + BPF_ALU64_REG(BPF_SUB, R1, R2), + BPF_ALU64_REG(BPF_MOV, R2, R1), + BPF_ALU64_IMM(BPF_RSH, R2, 32), + BPF_ALU64_REG(BPF_OR, R1, R2), + BPF_ALU64_REG(BPF_MOV, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + /* Checking that JMP32 immediate src is not zero extended in place */ +#define BPF_JMP32_IMM_ZEXT(op) \ + { \ + "JMP32_" #op "_K: operand preserved in zext", \ + .u.insns_int = { \ + BPF_LD_IMM64(R0, 0x0123456789acbdefULL),\ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_JMP32_IMM(BPF_##op, R0, 1234, 1), \ + BPF_JMP_A(0), /* Nop */ \ + BPF_ALU64_REG(BPF_SUB, R0, R1), \ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_ALU64_IMM(BPF_RSH, R1, 32), \ + BPF_ALU64_REG(BPF_OR, R0, R1), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, 0 } }, \ + } + BPF_JMP32_IMM_ZEXT(JEQ), + BPF_JMP32_IMM_ZEXT(JNE), + BPF_JMP32_IMM_ZEXT(JSET), + BPF_JMP32_IMM_ZEXT(JGT), + BPF_JMP32_IMM_ZEXT(JGE), + BPF_JMP32_IMM_ZEXT(JLT), + BPF_JMP32_IMM_ZEXT(JLE), + BPF_JMP32_IMM_ZEXT(JSGT), + BPF_JMP32_IMM_ZEXT(JSGE), + BPF_JMP32_IMM_ZEXT(JSGT), + BPF_JMP32_IMM_ZEXT(JSLT), + BPF_JMP32_IMM_ZEXT(JSLE), +#undef BPF_JMP2_IMM_ZEXT + /* Checking that JMP32 dst & src are not zero extended in place */ +#define BPF_JMP32_REG_ZEXT(op) \ + { \ + "JMP32_" #op "_X: operands preserved in zext", \ + .u.insns_int = { \ + BPF_LD_IMM64(R0, 0x0123456789acbdefULL),\ + BPF_LD_IMM64(R1, 0xfedcba9876543210ULL),\ + BPF_ALU64_REG(BPF_MOV, R2, R0), \ + BPF_ALU64_REG(BPF_MOV, R3, R1), \ + BPF_JMP32_IMM(BPF_##op, R0, R1, 1), \ + BPF_JMP_A(0), /* Nop */ \ + BPF_ALU64_REG(BPF_SUB, R0, R2), \ + BPF_ALU64_REG(BPF_SUB, R1, R3), \ + BPF_ALU64_REG(BPF_OR, R0, R1), \ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_ALU64_IMM(BPF_RSH, R1, 32), \ + BPF_ALU64_REG(BPF_OR, R0, R1), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, 0 } }, \ + } + BPF_JMP32_REG_ZEXT(JEQ), + BPF_JMP32_REG_ZEXT(JNE), + BPF_JMP32_REG_ZEXT(JSET), + BPF_JMP32_REG_ZEXT(JGT), + BPF_JMP32_REG_ZEXT(JGE), + BPF_JMP32_REG_ZEXT(JLT), + BPF_JMP32_REG_ZEXT(JLE), + BPF_JMP32_REG_ZEXT(JSGT), + BPF_JMP32_REG_ZEXT(JSGE), + BPF_JMP32_REG_ZEXT(JSGT), + BPF_JMP32_REG_ZEXT(JSLT), + BPF_JMP32_REG_ZEXT(JSLE), +#undef BPF_JMP2_REG_ZEXT /* Exhaustive test of ALU64 shift operations */ { "ALU64_LSH_K: all shift values", -- cgit v1.2.3 From e2f9797b3c7396065ca3bc9f223225ca63c1e2bd Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Fri, 1 Oct 2021 15:03:43 +0200 Subject: bpf/tests: Add more tests for ALU and ATOMIC register clobbering This patch expands the register-clobbering-during-function-call tests to cover more all ALU32/64 MUL, DIV and MOD operations and all ATOMIC operations. In short, if a JIT implements a complex operation with a call to an external function, it must make sure to save and restore all its caller-saved registers that may be clobbered by the call. Signed-off-by: Johan Almbladh Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20211001130348.3670534-6-johan.almbladh@anyfinetworks.com --- lib/test_bpf.c | 267 ++++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 197 insertions(+), 70 deletions(-) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index c7db90112ef0..201f34060eef 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -3744,76 +3744,6 @@ static struct bpf_test tests[] = { { }, { { 0, -1 } } }, - { - /* - * Register (non-)clobbering test, in the case where a 32-bit - * JIT implements complex ALU64 operations via function calls. - * If so, the function call must be invisible in the eBPF - * registers. The JIT must then save and restore relevant - * registers during the call. The following tests check that - * the eBPF registers retain their values after such a call. - */ - "INT: Register clobbering, R1 updated", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_ALU32_IMM(BPF_MOV, R1, 123456789), - BPF_ALU32_IMM(BPF_MOV, R2, 2), - BPF_ALU32_IMM(BPF_MOV, R3, 3), - BPF_ALU32_IMM(BPF_MOV, R4, 4), - BPF_ALU32_IMM(BPF_MOV, R5, 5), - BPF_ALU32_IMM(BPF_MOV, R6, 6), - BPF_ALU32_IMM(BPF_MOV, R7, 7), - BPF_ALU32_IMM(BPF_MOV, R8, 8), - BPF_ALU32_IMM(BPF_MOV, R9, 9), - BPF_ALU64_IMM(BPF_DIV, R1, 123456789), - BPF_JMP_IMM(BPF_JNE, R0, 0, 10), - BPF_JMP_IMM(BPF_JNE, R1, 1, 9), - BPF_JMP_IMM(BPF_JNE, R2, 2, 8), - BPF_JMP_IMM(BPF_JNE, R3, 3, 7), - BPF_JMP_IMM(BPF_JNE, R4, 4, 6), - BPF_JMP_IMM(BPF_JNE, R5, 5, 5), - BPF_JMP_IMM(BPF_JNE, R6, 6, 4), - BPF_JMP_IMM(BPF_JNE, R7, 7, 3), - BPF_JMP_IMM(BPF_JNE, R8, 8, 2), - BPF_JMP_IMM(BPF_JNE, R9, 9, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "INT: Register clobbering, R2 updated", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_ALU32_IMM(BPF_MOV, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R2, 2 * 123456789), - BPF_ALU32_IMM(BPF_MOV, R3, 3), - BPF_ALU32_IMM(BPF_MOV, R4, 4), - BPF_ALU32_IMM(BPF_MOV, R5, 5), - BPF_ALU32_IMM(BPF_MOV, R6, 6), - BPF_ALU32_IMM(BPF_MOV, R7, 7), - BPF_ALU32_IMM(BPF_MOV, R8, 8), - BPF_ALU32_IMM(BPF_MOV, R9, 9), - BPF_ALU64_IMM(BPF_DIV, R2, 123456789), - BPF_JMP_IMM(BPF_JNE, R0, 0, 10), - BPF_JMP_IMM(BPF_JNE, R1, 1, 9), - BPF_JMP_IMM(BPF_JNE, R2, 2, 8), - BPF_JMP_IMM(BPF_JNE, R3, 3, 7), - BPF_JMP_IMM(BPF_JNE, R4, 4, 6), - BPF_JMP_IMM(BPF_JNE, R5, 5, 5), - BPF_JMP_IMM(BPF_JNE, R6, 6, 4), - BPF_JMP_IMM(BPF_JNE, R7, 7, 3), - BPF_JMP_IMM(BPF_JNE, R8, 8, 2), - BPF_JMP_IMM(BPF_JNE, R9, 9, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, { /* * Test 32-bit JITs that implement complex ALU64 operations as @@ -10586,6 +10516,203 @@ static struct bpf_test tests[] = { {}, { { 0, 2 } }, }, + /* + * Register (non-)clobbering tests for the case where a JIT implements + * complex ALU or ATOMIC operations via function calls. If so, the + * function call must be transparent to the eBPF registers. The JIT + * must therefore save and restore relevant registers across the call. + * The following tests check that the eBPF registers retain their + * values after such an operation. Mainly intended for complex ALU + * and atomic operation, but we run it for all. You never know... + * + * Note that each operations should be tested twice with different + * destinations, to check preservation for all registers. + */ +#define BPF_TEST_CLOBBER_ALU(alu, op, dst, src) \ + { \ + #alu "_" #op " to " #dst ": no clobbering", \ + .u.insns_int = { \ + BPF_ALU64_IMM(BPF_MOV, R0, R0), \ + BPF_ALU64_IMM(BPF_MOV, R1, R1), \ + BPF_ALU64_IMM(BPF_MOV, R2, R2), \ + BPF_ALU64_IMM(BPF_MOV, R3, R3), \ + BPF_ALU64_IMM(BPF_MOV, R4, R4), \ + BPF_ALU64_IMM(BPF_MOV, R5, R5), \ + BPF_ALU64_IMM(BPF_MOV, R6, R6), \ + BPF_ALU64_IMM(BPF_MOV, R7, R7), \ + BPF_ALU64_IMM(BPF_MOV, R8, R8), \ + BPF_ALU64_IMM(BPF_MOV, R9, R9), \ + BPF_##alu(BPF_ ##op, dst, src), \ + BPF_ALU32_IMM(BPF_MOV, dst, dst), \ + BPF_JMP_IMM(BPF_JNE, R0, R0, 10), \ + BPF_JMP_IMM(BPF_JNE, R1, R1, 9), \ + BPF_JMP_IMM(BPF_JNE, R2, R2, 8), \ + BPF_JMP_IMM(BPF_JNE, R3, R3, 7), \ + BPF_JMP_IMM(BPF_JNE, R4, R4, 6), \ + BPF_JMP_IMM(BPF_JNE, R5, R5, 5), \ + BPF_JMP_IMM(BPF_JNE, R6, R6, 4), \ + BPF_JMP_IMM(BPF_JNE, R7, R7, 3), \ + BPF_JMP_IMM(BPF_JNE, R8, R8, 2), \ + BPF_JMP_IMM(BPF_JNE, R9, R9, 1), \ + BPF_ALU64_IMM(BPF_MOV, R0, 1), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, 1 } } \ + } + /* ALU64 operations, register clobbering */ + BPF_TEST_CLOBBER_ALU(ALU64_IMM, AND, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, AND, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, OR, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, OR, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, XOR, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, XOR, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, LSH, R8, 12), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, LSH, R9, 12), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, RSH, R8, 12), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, RSH, R9, 12), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, ARSH, R8, 12), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, ARSH, R9, 12), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, ADD, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, ADD, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, SUB, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, SUB, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, MUL, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, MUL, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, DIV, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, DIV, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, MOD, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, MOD, R9, 123456789), + /* ALU32 immediate operations, register clobbering */ + BPF_TEST_CLOBBER_ALU(ALU32_IMM, AND, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, AND, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, OR, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, OR, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, XOR, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, XOR, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, LSH, R8, 12), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, LSH, R9, 12), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, RSH, R8, 12), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, RSH, R9, 12), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, ARSH, R8, 12), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, ARSH, R9, 12), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, ADD, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, ADD, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, SUB, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, SUB, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, MUL, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, MUL, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, DIV, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, DIV, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, MOD, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, MOD, R9, 123456789), + /* ALU64 register operations, register clobbering */ + BPF_TEST_CLOBBER_ALU(ALU64_REG, AND, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, AND, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, OR, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, OR, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, XOR, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, XOR, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, LSH, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, LSH, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, RSH, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, RSH, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, ARSH, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, ARSH, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, ADD, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, ADD, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, SUB, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, SUB, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, MUL, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, MUL, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, DIV, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, DIV, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, MOD, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, MOD, R9, R1), + /* ALU32 register operations, register clobbering */ + BPF_TEST_CLOBBER_ALU(ALU32_REG, AND, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, AND, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, OR, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, OR, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, XOR, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, XOR, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, LSH, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, LSH, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, RSH, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, RSH, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, ARSH, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, ARSH, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, ADD, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, ADD, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, SUB, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, SUB, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, MUL, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, MUL, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, DIV, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, DIV, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, MOD, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, MOD, R9, R1), +#undef BPF_TEST_CLOBBER_ALU +#define BPF_TEST_CLOBBER_ATOMIC(width, op) \ + { \ + "Atomic_" #width " " #op ": no clobbering", \ + .u.insns_int = { \ + BPF_ALU64_IMM(BPF_MOV, R0, 0), \ + BPF_ALU64_IMM(BPF_MOV, R1, 1), \ + BPF_ALU64_IMM(BPF_MOV, R2, 2), \ + BPF_ALU64_IMM(BPF_MOV, R3, 3), \ + BPF_ALU64_IMM(BPF_MOV, R4, 4), \ + BPF_ALU64_IMM(BPF_MOV, R5, 5), \ + BPF_ALU64_IMM(BPF_MOV, R6, 6), \ + BPF_ALU64_IMM(BPF_MOV, R7, 7), \ + BPF_ALU64_IMM(BPF_MOV, R8, 8), \ + BPF_ALU64_IMM(BPF_MOV, R9, 9), \ + BPF_ST_MEM(width, R10, -8, \ + (op) == BPF_CMPXCHG ? 0 : \ + (op) & BPF_FETCH ? 1 : 0), \ + BPF_ATOMIC_OP(width, op, R10, R1, -8), \ + BPF_JMP_IMM(BPF_JNE, R0, 0, 10), \ + BPF_JMP_IMM(BPF_JNE, R1, 1, 9), \ + BPF_JMP_IMM(BPF_JNE, R2, 2, 8), \ + BPF_JMP_IMM(BPF_JNE, R3, 3, 7), \ + BPF_JMP_IMM(BPF_JNE, R4, 4, 6), \ + BPF_JMP_IMM(BPF_JNE, R5, 5, 5), \ + BPF_JMP_IMM(BPF_JNE, R6, 6, 4), \ + BPF_JMP_IMM(BPF_JNE, R7, 7, 3), \ + BPF_JMP_IMM(BPF_JNE, R8, 8, 2), \ + BPF_JMP_IMM(BPF_JNE, R9, 9, 1), \ + BPF_ALU64_IMM(BPF_MOV, R0, 1), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, 1 } }, \ + .stack_depth = 8, \ + } + /* 64-bit atomic operations, register clobbering */ + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_ADD), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_AND), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_OR), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_XOR), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_ADD | BPF_FETCH), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_AND | BPF_FETCH), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_OR | BPF_FETCH), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_XOR | BPF_FETCH), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_XCHG), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_CMPXCHG), + /* 32-bit atomic operations, register clobbering */ + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_ADD), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_AND), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_OR), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_XOR), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_ADD | BPF_FETCH), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_AND | BPF_FETCH), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_OR | BPF_FETCH), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_XOR | BPF_FETCH), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_XCHG), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_CMPXCHG), +#undef BPF_TEST_CLOBBER_ATOMIC /* Checking that ALU32 src is not zero extended in place */ #define BPF_ALU32_SRC_ZEXT(op) \ { \ -- cgit v1.2.3 From e42fc3c2c40e0fb9d371c146dc8c0a70bee88a3c Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Fri, 1 Oct 2021 15:03:44 +0200 Subject: bpf/tests: Minor restructuring of ALU tests This patch moves the ALU LSH/RSH/ARSH reference computations into the common reference value function. Also fix typo in constants so they now have the intended values. Signed-off-by: Johan Almbladh Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20211001130348.3670534-7-johan.almbladh@anyfinetworks.com --- lib/test_bpf.c | 137 +++++++++++++++++++++++++++------------------------------ 1 file changed, 65 insertions(+), 72 deletions(-) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index 201f34060eef..919323a3b69f 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -538,6 +538,57 @@ static int bpf_fill_max_jmp_never_taken(struct bpf_test *self) return __bpf_fill_max_jmp(self, BPF_JLT, 0); } +/* ALU result computation used in tests */ +static bool __bpf_alu_result(u64 *res, u64 v1, u64 v2, u8 op) +{ + *res = 0; + switch (op) { + case BPF_MOV: + *res = v2; + break; + case BPF_AND: + *res = v1 & v2; + break; + case BPF_OR: + *res = v1 | v2; + break; + case BPF_XOR: + *res = v1 ^ v2; + break; + case BPF_LSH: + *res = v1 << v2; + break; + case BPF_RSH: + *res = v1 >> v2; + break; + case BPF_ARSH: + *res = v1 >> v2; + if (v2 > 0 && v1 > S64_MAX) + *res |= ~0ULL << (64 - v2); + break; + case BPF_ADD: + *res = v1 + v2; + break; + case BPF_SUB: + *res = v1 - v2; + break; + case BPF_MUL: + *res = v1 * v2; + break; + case BPF_DIV: + if (v2 == 0) + return false; + *res = div64_u64(v1, v2); + break; + case BPF_MOD: + if (v2 == 0) + return false; + div64_u64_rem(v1, v2, res); + break; + } + return true; +} + /* Test an ALU shift operation for all valid shift values */ static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op, u8 mode, bool alu32) @@ -576,37 +627,19 @@ static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op, insn[i++] = BPF_ALU32_IMM(op, R1, imm); else insn[i++] = BPF_ALU32_REG(op, R1, R2); - switch (op) { - case BPF_LSH: - val = (u32)reg << imm; - break; - case BPF_RSH: - val = (u32)reg >> imm; - break; - case BPF_ARSH: - val = (u32)reg >> imm; - if (imm > 0 && (reg & 0x80000000)) - val |= ~(u32)0 << (32 - imm); - break; - } + + if (op == BPF_ARSH) + reg = (s32)reg; + else + reg = (u32)reg; + __bpf_alu_result(&val, reg, imm, op); + val = (u32)val; } else { if (mode == BPF_K) insn[i++] = BPF_ALU64_IMM(op, R1, imm); else insn[i++] = BPF_ALU64_REG(op, R1, R2); - switch (op) { - case BPF_LSH: - val = (u64)reg << imm; - break; - case BPF_RSH: - val = (u64)reg >> imm; - break; - case BPF_ARSH: - val = (u64)reg >> imm; - if (imm > 0 && reg < 0) - val |= ~(u64)0 << (64 - imm); - break; - } + __bpf_alu_result(&val, reg, imm, op); } /* @@ -799,46 +832,6 @@ static int __bpf_fill_pattern(struct bpf_test *self, void *arg, * test is designed to verify e.g. the ALU and ALU64 operations for JITs that * emit different code depending on the magnitude of the immediate value. */ - -static bool __bpf_alu_result(u64 *res, u64 v1, u64 v2, u8 op) -{ - *res = 0; - switch (op) { - case BPF_MOV: - *res = v2; - break; - case BPF_AND: - *res = v1 & v2; - break; - case BPF_OR: - *res = v1 | v2; - break; - case BPF_XOR: - *res = v1 ^ v2; - break; - case BPF_ADD: - *res = v1 + v2; - break; - case BPF_SUB: - *res = v1 - v2; - break; - case BPF_MUL: - *res = v1 * v2; - break; - case BPF_DIV: - if (v2 == 0) - return false; - *res = div64_u64(v1, v2); - break; - case BPF_MOD: - if (v2 == 0) - return false; - div64_u64_rem(v1, v2, res); - break; - } - return true; -} - static int __bpf_emit_alu64_imm(struct bpf_test *self, void *arg, struct bpf_insn *insns, s64 dst, s64 imm) { @@ -7881,7 +7874,7 @@ static struct bpf_test tests[] = { "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test successful return", .u.insns_int = { BPF_LD_IMM64(R1, 0x0123456789abcdefULL), - BPF_LD_IMM64(R2, 0xfecdba9876543210ULL), + BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), BPF_ALU64_REG(BPF_MOV, R0, R1), BPF_STX_MEM(BPF_DW, R10, R1, -40), BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), @@ -7898,7 +7891,7 @@ static struct bpf_test tests[] = { "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test successful store", .u.insns_int = { BPF_LD_IMM64(R1, 0x0123456789abcdefULL), - BPF_LD_IMM64(R2, 0xfecdba9876543210ULL), + BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), BPF_ALU64_REG(BPF_MOV, R0, R1), BPF_STX_MEM(BPF_DW, R10, R0, -40), BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), @@ -7916,7 +7909,7 @@ static struct bpf_test tests[] = { "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test failure return", .u.insns_int = { BPF_LD_IMM64(R1, 0x0123456789abcdefULL), - BPF_LD_IMM64(R2, 0xfecdba9876543210ULL), + BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), BPF_ALU64_REG(BPF_MOV, R0, R1), BPF_ALU64_IMM(BPF_ADD, R0, 1), BPF_STX_MEM(BPF_DW, R10, R1, -40), @@ -7934,7 +7927,7 @@ static struct bpf_test tests[] = { "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test failure store", .u.insns_int = { BPF_LD_IMM64(R1, 0x0123456789abcdefULL), - BPF_LD_IMM64(R2, 0xfecdba9876543210ULL), + BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), BPF_ALU64_REG(BPF_MOV, R0, R1), BPF_ALU64_IMM(BPF_ADD, R0, 1), BPF_STX_MEM(BPF_DW, R10, R1, -40), @@ -7953,11 +7946,11 @@ static struct bpf_test tests[] = { "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test side effects", .u.insns_int = { BPF_LD_IMM64(R1, 0x0123456789abcdefULL), - BPF_LD_IMM64(R2, 0xfecdba9876543210ULL), + BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), BPF_ALU64_REG(BPF_MOV, R0, R1), BPF_STX_MEM(BPF_DW, R10, R1, -40), BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), - BPF_LD_IMM64(R0, 0xfecdba9876543210ULL), + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), BPF_JMP_REG(BPF_JNE, R0, R2, 1), BPF_ALU64_REG(BPF_SUB, R0, R2), BPF_EXIT_INSN(), -- cgit v1.2.3 From daed6083f4fbcbb57da26d80f15365219ae793de Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Fri, 1 Oct 2021 15:03:45 +0200 Subject: bpf/tests: Add exhaustive tests of ALU register combinations This patch replaces the current register combination test with new exhaustive tests. Before, only a subset of register combinations was tested for ALU64 DIV. Now, all combinatons of operand registers are tested, including the case when they are the same, and for all ALU32 and ALU64 operations. Signed-off-by: Johan Almbladh Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20211001130348.3670534-8-johan.almbladh@anyfinetworks.com --- lib/test_bpf.c | 834 ++++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 763 insertions(+), 71 deletions(-) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index 919323a3b69f..924bf4c9783c 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -1130,6 +1130,381 @@ static int bpf_fill_alu32_mod_reg(struct bpf_test *self) return __bpf_fill_alu32_reg(self, BPF_MOD); } +/* + * Test JITs that implement complex ALU operations as function + * calls, and must re-arrange operands for argument passing. + */ +static int __bpf_fill_alu_imm_regs(struct bpf_test *self, u8 op, bool alu32) +{ + int len = 2 + 10 * 10; + struct bpf_insn *insns; + u64 dst, res; + int i = 0; + u32 imm; + int rd; + + insns = kmalloc_array(len, sizeof(*insns), GFP_KERNEL); + if (!insns) + return -ENOMEM; + + /* Operand and result values according to operation */ + if (alu32) + dst = 0x76543210U; + else + dst = 0x7edcba9876543210ULL; + imm = 0x01234567U; + + if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH) + imm &= 31; + + __bpf_alu_result(&res, dst, imm, op); + + if (alu32) + res = (u32)res; + + /* Check all operand registers */ + for (rd = R0; rd <= R9; rd++) { + i += __bpf_ld_imm64(&insns[i], rd, dst); + + if (alu32) + insns[i++] = BPF_ALU32_IMM(op, rd, imm); + else + insns[i++] = BPF_ALU64_IMM(op, rd, imm); + + insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, res, 2); + insns[i++] = BPF_MOV64_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_ALU64_IMM(BPF_RSH, rd, 32); + insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, res >> 32, 2); + insns[i++] = BPF_MOV64_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + } + + insns[i++] = BPF_MOV64_IMM(R0, 1); + insns[i++] = BPF_EXIT_INSN(); + + self->u.ptr.insns = insns; + self->u.ptr.len = len; + BUG_ON(i != len); + + return 0; +} + +/* ALU64 K registers */ +static int bpf_fill_alu64_mov_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_MOV, false); +} + +static int bpf_fill_alu64_and_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_AND, false); +} + +static int bpf_fill_alu64_or_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_OR, false); +} + +static int bpf_fill_alu64_xor_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_XOR, false); +} + +static int bpf_fill_alu64_lsh_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_LSH, false); +} + +static int bpf_fill_alu64_rsh_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_RSH, false); +} + +static int bpf_fill_alu64_arsh_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_ARSH, false); +} + +static int bpf_fill_alu64_add_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_ADD, false); +} + +static int bpf_fill_alu64_sub_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_SUB, false); +} + +static int bpf_fill_alu64_mul_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_MUL, false); +} + +static int bpf_fill_alu64_div_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_DIV, false); +} + +static int bpf_fill_alu64_mod_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_MOD, false); +} + +/* ALU32 K registers */ +static int bpf_fill_alu32_mov_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_MOV, true); +} + +static int bpf_fill_alu32_and_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_AND, true); +} + +static int bpf_fill_alu32_or_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_OR, true); +} + +static int bpf_fill_alu32_xor_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_XOR, true); +} + +static int bpf_fill_alu32_lsh_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_LSH, true); +} + +static int bpf_fill_alu32_rsh_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_RSH, true); +} + +static int bpf_fill_alu32_arsh_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_ARSH, true); +} + +static int bpf_fill_alu32_add_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_ADD, true); +} + +static int bpf_fill_alu32_sub_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_SUB, true); +} + +static int bpf_fill_alu32_mul_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_MUL, true); +} + +static int bpf_fill_alu32_div_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_DIV, true); +} + +static int bpf_fill_alu32_mod_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_MOD, true); +} + +/* + * Test JITs that implement complex ALU operations as function + * calls, and must re-arrange operands for argument passing. + */ +static int __bpf_fill_alu_reg_pairs(struct bpf_test *self, u8 op, bool alu32) +{ + int len = 2 + 10 * 10 * 12; + u64 dst, src, res, same; + struct bpf_insn *insns; + int rd, rs; + int i = 0; + + insns = kmalloc_array(len, sizeof(*insns), GFP_KERNEL); + if (!insns) + return -ENOMEM; + + /* Operand and result values according to operation */ + if (alu32) { + dst = 0x76543210U; + src = 0x01234567U; + } else { + dst = 0x7edcba9876543210ULL; + src = 0x0123456789abcdefULL; + } + + if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH) + src &= 31; + + __bpf_alu_result(&res, dst, src, op); + __bpf_alu_result(&same, src, src, op); + + if (alu32) { + res = (u32)res; + same = (u32)same; + } + + /* Check all combinations of operand registers */ + for (rd = R0; rd <= R9; rd++) { + for (rs = R0; rs <= R9; rs++) { + u64 val = rd == rs ? same : res; + + i += __bpf_ld_imm64(&insns[i], rd, dst); + i += __bpf_ld_imm64(&insns[i], rs, src); + + if (alu32) + insns[i++] = BPF_ALU32_REG(op, rd, rs); + else + insns[i++] = BPF_ALU64_REG(op, rd, rs); + + insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, val, 2); + insns[i++] = BPF_MOV64_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_ALU64_IMM(BPF_RSH, rd, 32); + insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, val >> 32, 2); + insns[i++] = BPF_MOV64_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + } + } + + insns[i++] = BPF_MOV64_IMM(R0, 1); + insns[i++] = BPF_EXIT_INSN(); + + self->u.ptr.insns = insns; + self->u.ptr.len = len; + BUG_ON(i != len); + + return 0; +} + +/* ALU64 X register combinations */ +static int bpf_fill_alu64_mov_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_MOV, false); +} + +static int bpf_fill_alu64_and_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_AND, false); +} + +static int bpf_fill_alu64_or_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_OR, false); +} + +static int bpf_fill_alu64_xor_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_XOR, false); +} + +static int bpf_fill_alu64_lsh_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_LSH, false); +} + +static int bpf_fill_alu64_rsh_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_RSH, false); +} + +static int bpf_fill_alu64_arsh_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_ARSH, false); +} + +static int bpf_fill_alu64_add_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_ADD, false); +} + +static int bpf_fill_alu64_sub_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_SUB, false); +} + +static int bpf_fill_alu64_mul_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_MUL, false); +} + +static int bpf_fill_alu64_div_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_DIV, false); +} + +static int bpf_fill_alu64_mod_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_MOD, false); +} + +/* ALU32 X register combinations */ +static int bpf_fill_alu32_mov_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_MOV, true); +} + +static int bpf_fill_alu32_and_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_AND, true); +} + +static int bpf_fill_alu32_or_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_OR, true); +} + +static int bpf_fill_alu32_xor_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_XOR, true); +} + +static int bpf_fill_alu32_lsh_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_LSH, true); +} + +static int bpf_fill_alu32_rsh_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_RSH, true); +} + +static int bpf_fill_alu32_arsh_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_ARSH, true); +} + +static int bpf_fill_alu32_add_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_ADD, true); +} + +static int bpf_fill_alu32_sub_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_SUB, true); +} + +static int bpf_fill_alu32_mul_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_MUL, true); +} + +static int bpf_fill_alu32_div_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_DIV, true); +} + +static int bpf_fill_alu32_mod_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_MOD, true); +} + /* * Exhaustive tests of atomic operations for all power-of-two operand * magnitudes, both for positive and negative values. @@ -3737,77 +4112,6 @@ static struct bpf_test tests[] = { { }, { { 0, -1 } } }, - { - /* - * Test 32-bit JITs that implement complex ALU64 operations as - * function calls R0 = f(R1, R2), and must re-arrange operands. - */ -#define NUMER 0xfedcba9876543210ULL -#define DENOM 0x0123456789abcdefULL - "ALU64_DIV X: Operand register permutations", - .u.insns_int = { - /* R0 / R2 */ - BPF_LD_IMM64(R0, NUMER), - BPF_LD_IMM64(R2, DENOM), - BPF_ALU64_REG(BPF_DIV, R0, R2), - BPF_JMP_IMM(BPF_JEQ, R0, NUMER / DENOM, 1), - BPF_EXIT_INSN(), - /* R1 / R0 */ - BPF_LD_IMM64(R1, NUMER), - BPF_LD_IMM64(R0, DENOM), - BPF_ALU64_REG(BPF_DIV, R1, R0), - BPF_JMP_IMM(BPF_JEQ, R1, NUMER / DENOM, 1), - BPF_EXIT_INSN(), - /* R0 / R1 */ - BPF_LD_IMM64(R0, NUMER), - BPF_LD_IMM64(R1, DENOM), - BPF_ALU64_REG(BPF_DIV, R0, R1), - BPF_JMP_IMM(BPF_JEQ, R0, NUMER / DENOM, 1), - BPF_EXIT_INSN(), - /* R2 / R0 */ - BPF_LD_IMM64(R2, NUMER), - BPF_LD_IMM64(R0, DENOM), - BPF_ALU64_REG(BPF_DIV, R2, R0), - BPF_JMP_IMM(BPF_JEQ, R2, NUMER / DENOM, 1), - BPF_EXIT_INSN(), - /* R2 / R1 */ - BPF_LD_IMM64(R2, NUMER), - BPF_LD_IMM64(R1, DENOM), - BPF_ALU64_REG(BPF_DIV, R2, R1), - BPF_JMP_IMM(BPF_JEQ, R2, NUMER / DENOM, 1), - BPF_EXIT_INSN(), - /* R1 / R2 */ - BPF_LD_IMM64(R1, NUMER), - BPF_LD_IMM64(R2, DENOM), - BPF_ALU64_REG(BPF_DIV, R1, R2), - BPF_JMP_IMM(BPF_JEQ, R1, NUMER / DENOM, 1), - BPF_EXIT_INSN(), - /* R1 / R1 */ - BPF_LD_IMM64(R1, NUMER), - BPF_ALU64_REG(BPF_DIV, R1, R1), - BPF_JMP_IMM(BPF_JEQ, R1, 1, 1), - BPF_EXIT_INSN(), - /* R2 / R2 */ - BPF_LD_IMM64(R2, DENOM), - BPF_ALU64_REG(BPF_DIV, R2, R2), - BPF_JMP_IMM(BPF_JEQ, R2, 1, 1), - BPF_EXIT_INSN(), - /* R3 / R4 */ - BPF_LD_IMM64(R3, NUMER), - BPF_LD_IMM64(R4, DENOM), - BPF_ALU64_REG(BPF_DIV, R3, R4), - BPF_JMP_IMM(BPF_JEQ, R3, NUMER / DENOM, 1), - BPF_EXIT_INSN(), - /* Successful return */ - BPF_LD_IMM64(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, -#undef NUMER -#undef DENOM - }, #ifdef CONFIG_32BIT { "INT: 32-bit context pointer word order and zero-extension", @@ -10849,6 +11153,394 @@ static struct bpf_test tests[] = { BPF_JMP32_REG_ZEXT(JSLT), BPF_JMP32_REG_ZEXT(JSLE), #undef BPF_JMP2_REG_ZEXT + /* ALU64 K register combinations */ + { + "ALU64_MOV_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mov_imm_regs, + }, + { + "ALU64_AND_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_and_imm_regs, + }, + { + "ALU64_OR_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_or_imm_regs, + }, + { + "ALU64_XOR_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_xor_imm_regs, + }, + { + "ALU64_LSH_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_lsh_imm_regs, + }, + { + "ALU64_RSH_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_rsh_imm_regs, + }, + { + "ALU64_ARSH_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_arsh_imm_regs, + }, + { + "ALU64_ADD_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_add_imm_regs, + }, + { + "ALU64_SUB_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_sub_imm_regs, + }, + { + "ALU64_MUL_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mul_imm_regs, + }, + { + "ALU64_DIV_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_div_imm_regs, + }, + { + "ALU64_MOD_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mod_imm_regs, + }, + /* ALU32 K registers */ + { + "ALU32_MOV_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mov_imm_regs, + }, + { + "ALU32_AND_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_and_imm_regs, + }, + { + "ALU32_OR_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_or_imm_regs, + }, + { + "ALU32_XOR_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_xor_imm_regs, + }, + { + "ALU32_LSH_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_lsh_imm_regs, + }, + { + "ALU32_RSH_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_rsh_imm_regs, + }, + { + "ALU32_ARSH_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_arsh_imm_regs, + }, + { + "ALU32_ADD_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_add_imm_regs, + }, + { + "ALU32_SUB_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_sub_imm_regs, + }, + { + "ALU32_MUL_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mul_imm_regs, + }, + { + "ALU32_DIV_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_div_imm_regs, + }, + { + "ALU32_MOD_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mod_imm_regs, + }, + /* ALU64 X register combinations */ + { + "ALU64_MOV_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mov_reg_pairs, + }, + { + "ALU64_AND_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_and_reg_pairs, + }, + { + "ALU64_OR_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_or_reg_pairs, + }, + { + "ALU64_XOR_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_xor_reg_pairs, + }, + { + "ALU64_LSH_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_lsh_reg_pairs, + }, + { + "ALU64_RSH_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_rsh_reg_pairs, + }, + { + "ALU64_ARSH_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_arsh_reg_pairs, + }, + { + "ALU64_ADD_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_add_reg_pairs, + }, + { + "ALU64_SUB_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_sub_reg_pairs, + }, + { + "ALU64_MUL_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mul_reg_pairs, + }, + { + "ALU64_DIV_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_div_reg_pairs, + }, + { + "ALU64_MOD_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mod_reg_pairs, + }, + /* ALU32 X register combinations */ + { + "ALU32_MOV_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mov_reg_pairs, + }, + { + "ALU32_AND_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_and_reg_pairs, + }, + { + "ALU32_OR_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_or_reg_pairs, + }, + { + "ALU32_XOR_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_xor_reg_pairs, + }, + { + "ALU32_LSH_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_lsh_reg_pairs, + }, + { + "ALU32_RSH_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_rsh_reg_pairs, + }, + { + "ALU32_ARSH_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_arsh_reg_pairs, + }, + { + "ALU32_ADD_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_add_reg_pairs, + }, + { + "ALU32_SUB_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_sub_reg_pairs, + }, + { + "ALU32_MUL_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mul_reg_pairs, + }, + { + "ALU32_DIV_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_div_reg_pairs, + }, + { + "ALU32_MOD_X register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mod_reg_pairs, + }, /* Exhaustive test of ALU64 shift operations */ { "ALU64_LSH_K: all shift values", -- cgit v1.2.3 From 6fae2e8a1d9ee09e25aee6514b7544e059a8ee68 Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Fri, 1 Oct 2021 15:03:46 +0200 Subject: bpf/tests: Add exhaustive tests of BPF_ATOMIC register combinations This patch adds tests of all register combinations for BPF_ATOMIC operations on both BPF_W and BPF_DW sizes. Signed-off-by: Johan Almbladh Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20211001130348.3670534-9-johan.almbladh@anyfinetworks.com --- lib/test_bpf.c | 422 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 422 insertions(+) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index 924bf4c9783c..40db4cee4f51 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -1805,6 +1805,246 @@ static int bpf_fill_cmpxchg32(struct bpf_test *self) &__bpf_emit_cmpxchg32); } +/* + * Test JITs that implement ATOMIC operations as function calls or + * other primitives, and must re-arrange operands for argument passing. + */ +static int __bpf_fill_atomic_reg_pairs(struct bpf_test *self, u8 width, u8 op) +{ + struct bpf_insn *insn; + int len = 2 + 34 * 10 * 10; + u64 mem, upd, res; + int rd, rs, i = 0; + + insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL); + if (!insn) + return -ENOMEM; + + /* Operand and memory values */ + if (width == BPF_DW) { + mem = 0x0123456789abcdefULL; + upd = 0xfedcba9876543210ULL; + } else { /* BPF_W */ + mem = 0x01234567U; + upd = 0x76543210U; + } + + /* Memory updated according to operation */ + switch (op) { + case BPF_XCHG: + res = upd; + break; + case BPF_CMPXCHG: + res = mem; + break; + default: + __bpf_alu_result(&res, mem, upd, BPF_OP(op)); + } + + /* Test all operand registers */ + for (rd = R0; rd <= R9; rd++) { + for (rs = R0; rs <= R9; rs++) { + u64 cmp, src; + + /* Initialize value in memory */ + i += __bpf_ld_imm64(&insn[i], R0, mem); + insn[i++] = BPF_STX_MEM(width, R10, R0, -8); + + /* Initialize registers in order */ + i += __bpf_ld_imm64(&insn[i], R0, ~mem); + i += __bpf_ld_imm64(&insn[i], rs, upd); + insn[i++] = BPF_MOV64_REG(rd, R10); + + /* Perform atomic operation */ + insn[i++] = BPF_ATOMIC_OP(width, op, rd, rs, -8); + if (op == BPF_CMPXCHG && width == BPF_W) + insn[i++] = BPF_ZEXT_REG(R0); + + /* Check R0 register value */ + if (op == BPF_CMPXCHG) + cmp = mem; /* Expect value from memory */ + else if (R0 == rd || R0 == rs) + cmp = 0; /* Aliased, checked below */ + else + cmp = ~mem; /* Expect value to be preserved */ + if (cmp) { + insn[i++] = BPF_JMP32_IMM(BPF_JEQ, R0, + (u32)cmp, 2); + insn[i++] = BPF_MOV32_IMM(R0, __LINE__); + insn[i++] = BPF_EXIT_INSN(); + insn[i++] = BPF_ALU64_IMM(BPF_RSH, R0, 32); + insn[i++] = BPF_JMP32_IMM(BPF_JEQ, R0, + cmp >> 32, 2); + insn[i++] = BPF_MOV32_IMM(R0, __LINE__); + insn[i++] = BPF_EXIT_INSN(); + } + + /* Check source register value */ + if (rs == R0 && op == BPF_CMPXCHG) + src = 0; /* Aliased with R0, checked above */ + else if (rs == rd && (op == BPF_CMPXCHG || + !(op & BPF_FETCH))) + src = 0; /* Aliased with rd, checked below */ + else if (op == BPF_CMPXCHG) + src = upd; /* Expect value to be preserved */ + else if (op & BPF_FETCH) + src = mem; /* Expect fetched value from mem */ + else /* no fetch */ + src = upd; /* Expect value to be preserved */ + if (src) { + insn[i++] = BPF_JMP32_IMM(BPF_JEQ, rs, + (u32)src, 2); + insn[i++] = BPF_MOV32_IMM(R0, __LINE__); + insn[i++] = BPF_EXIT_INSN(); + insn[i++] = BPF_ALU64_IMM(BPF_RSH, rs, 32); + insn[i++] = BPF_JMP32_IMM(BPF_JEQ, rs, + src >> 32, 2); + insn[i++] = BPF_MOV32_IMM(R0, __LINE__); + insn[i++] = BPF_EXIT_INSN(); + } + + /* Check destination register value */ + if (!(rd == R0 && op == BPF_CMPXCHG) && + !(rd == rs && (op & BPF_FETCH))) { + insn[i++] = BPF_JMP_REG(BPF_JEQ, rd, R10, 2); + insn[i++] = BPF_MOV32_IMM(R0, __LINE__); + insn[i++] = BPF_EXIT_INSN(); + } + + /* Check value in memory */ + if (rs != rd) { /* No aliasing */ + i += __bpf_ld_imm64(&insn[i], R1, res); + } else if (op == BPF_XCHG) { /* Aliased, XCHG */ + insn[i++] = BPF_MOV64_REG(R1, R10); + } else if (op == BPF_CMPXCHG) { /* Aliased, CMPXCHG */ + i += __bpf_ld_imm64(&insn[i], R1, mem); + } else { /* Aliased, ALU oper */ + i += __bpf_ld_imm64(&insn[i], R1, mem); + insn[i++] = BPF_ALU64_REG(BPF_OP(op), R1, R10); + } + + insn[i++] = BPF_LDX_MEM(width, R0, R10, -8); + if (width == BPF_DW) + insn[i++] = BPF_JMP_REG(BPF_JEQ, R0, R1, 2); + else /* width == BPF_W */ + insn[i++] = BPF_JMP32_REG(BPF_JEQ, R0, R1, 2); + insn[i++] = BPF_MOV32_IMM(R0, __LINE__); + insn[i++] = BPF_EXIT_INSN(); + } + } + + insn[i++] = BPF_MOV64_IMM(R0, 1); + insn[i++] = BPF_EXIT_INSN(); + + self->u.ptr.insns = insn; + self->u.ptr.len = i; + BUG_ON(i > len); + + return 0; +} + +/* 64-bit atomic register tests */ +static int bpf_fill_atomic64_add_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_ADD); +} + +static int bpf_fill_atomic64_and_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_AND); +} + +static int bpf_fill_atomic64_or_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_OR); +} + +static int bpf_fill_atomic64_xor_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_XOR); +} + +static int bpf_fill_atomic64_add_fetch_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_ADD | BPF_FETCH); +} + +static int bpf_fill_atomic64_and_fetch_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_AND | BPF_FETCH); +} + +static int bpf_fill_atomic64_or_fetch_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_OR | BPF_FETCH); +} + +static int bpf_fill_atomic64_xor_fetch_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_XOR | BPF_FETCH); +} + +static int bpf_fill_atomic64_xchg_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_XCHG); +} + +static int bpf_fill_atomic64_cmpxchg_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_CMPXCHG); +} + +/* 32-bit atomic register tests */ +static int bpf_fill_atomic32_add_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_ADD); +} + +static int bpf_fill_atomic32_and_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_AND); +} + +static int bpf_fill_atomic32_or_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_OR); +} + +static int bpf_fill_atomic32_xor_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_XOR); +} + +static int bpf_fill_atomic32_add_fetch_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_ADD | BPF_FETCH); +} + +static int bpf_fill_atomic32_and_fetch_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_AND | BPF_FETCH); +} + +static int bpf_fill_atomic32_or_fetch_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_OR | BPF_FETCH); +} + +static int bpf_fill_atomic32_xor_fetch_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_XOR | BPF_FETCH); +} + +static int bpf_fill_atomic32_xchg_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_XCHG); +} + +static int bpf_fill_atomic32_cmpxchg_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_CMPXCHG); +} + /* * Test the two-instruction 64-bit immediate load operation for all * power-of-two magnitudes of the immediate operand. For each MSB, a block @@ -11976,6 +12216,188 @@ static struct bpf_test tests[] = { { { 0, 1 } }, .fill_helper = bpf_fill_ld_imm64, }, + /* 64-bit ATOMIC register combinations */ + { + "ATOMIC_DW_ADD: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_add_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_AND: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_and_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_OR: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_or_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_XOR: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_xor_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_ADD_FETCH: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_add_fetch_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_AND_FETCH: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_and_fetch_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_OR_FETCH: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_or_fetch_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_XOR_FETCH: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_xor_fetch_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_XCHG: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_xchg_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_CMPXCHG: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_cmpxchg_reg_pairs, + .stack_depth = 8, + }, + /* 32-bit ATOMIC register combinations */ + { + "ATOMIC_W_ADD: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_add_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_AND: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_and_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_OR: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_or_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_XOR: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_xor_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_ADD_FETCH: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_add_fetch_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_AND_FETCH: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_and_fetch_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_OR_FETCH: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_or_fetch_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_XOR_FETCH: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_xor_fetch_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_XCHG: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_xchg_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_CMPXCHG: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_cmpxchg_reg_pairs, + .stack_depth = 8, + }, /* 64-bit ATOMIC magnitudes */ { "ATOMIC_DW_ADD: all operand magnitudes", -- cgit v1.2.3 From 68813605dea69ca0af26af7dd00384c78c1df05d Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Fri, 1 Oct 2021 15:03:47 +0200 Subject: bpf/tests: Add test of ALU shifts with operand register aliasing This patch adds a tests of ALU32 and ALU64 LSH/RSH/ARSH operations for the case when the two operands are the same register. Mainly intended to test JITs that implement ALU64 shifts with 32-bit CPU instructions. Also renamed related helper functions for consistency with the new tests. Signed-off-by: Johan Almbladh Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20211001130348.3670534-10-johan.almbladh@anyfinetworks.com --- lib/test_bpf.c | 162 ++++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 149 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index 40db4cee4f51..dfcbdff714b6 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -660,37 +660,37 @@ static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op, self->u.ptr.insns = insn; self->u.ptr.len = len; - BUG_ON(i > len); + BUG_ON(i != len); return 0; } -static int bpf_fill_alu_lsh_imm(struct bpf_test *self) +static int bpf_fill_alu64_lsh_imm(struct bpf_test *self) { return __bpf_fill_alu_shift(self, BPF_LSH, BPF_K, false); } -static int bpf_fill_alu_rsh_imm(struct bpf_test *self) +static int bpf_fill_alu64_rsh_imm(struct bpf_test *self) { return __bpf_fill_alu_shift(self, BPF_RSH, BPF_K, false); } -static int bpf_fill_alu_arsh_imm(struct bpf_test *self) +static int bpf_fill_alu64_arsh_imm(struct bpf_test *self) { return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_K, false); } -static int bpf_fill_alu_lsh_reg(struct bpf_test *self) +static int bpf_fill_alu64_lsh_reg(struct bpf_test *self) { return __bpf_fill_alu_shift(self, BPF_LSH, BPF_X, false); } -static int bpf_fill_alu_rsh_reg(struct bpf_test *self) +static int bpf_fill_alu64_rsh_reg(struct bpf_test *self) { return __bpf_fill_alu_shift(self, BPF_RSH, BPF_X, false); } -static int bpf_fill_alu_arsh_reg(struct bpf_test *self) +static int bpf_fill_alu64_arsh_reg(struct bpf_test *self) { return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, false); } @@ -725,6 +725,86 @@ static int bpf_fill_alu32_arsh_reg(struct bpf_test *self) return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, true); } +/* + * Test an ALU register shift operation for all valid shift values + * for the case when the source and destination are the same. + */ +static int __bpf_fill_alu_shift_same_reg(struct bpf_test *self, u8 op, + bool alu32) +{ + int bits = alu32 ? 32 : 64; + int len = 3 + 6 * bits; + struct bpf_insn *insn; + int i = 0; + u64 val; + + insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL); + if (!insn) + return -ENOMEM; + + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); + + for (val = 0; val < bits; val++) { + u64 res; + + /* Perform operation */ + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R1, val); + if (alu32) + insn[i++] = BPF_ALU32_REG(op, R1, R1); + else + insn[i++] = BPF_ALU64_REG(op, R1, R1); + + /* Compute the reference result */ + __bpf_alu_result(&res, val, val, op); + if (alu32) + res = (u32)res; + i += __bpf_ld_imm64(&insn[i], R2, res); + + /* Check the actual result */ + insn[i++] = BPF_JMP_REG(BPF_JEQ, R1, R2, 1); + insn[i++] = BPF_EXIT_INSN(); + } + + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); + insn[i++] = BPF_EXIT_INSN(); + + self->u.ptr.insns = insn; + self->u.ptr.len = len; + BUG_ON(i != len); + + return 0; +} + +static int bpf_fill_alu64_lsh_same_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift_same_reg(self, BPF_LSH, false); +} + +static int bpf_fill_alu64_rsh_same_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift_same_reg(self, BPF_RSH, false); +} + +static int bpf_fill_alu64_arsh_same_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift_same_reg(self, BPF_ARSH, false); +} + +static int bpf_fill_alu32_lsh_same_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift_same_reg(self, BPF_LSH, true); +} + +static int bpf_fill_alu32_rsh_same_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift_same_reg(self, BPF_RSH, true); +} + +static int bpf_fill_alu32_arsh_same_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift_same_reg(self, BPF_ARSH, true); +} + /* * Common operand pattern generator for exhaustive power-of-two magnitudes * tests. The block size parameters can be adjusted to increase/reduce the @@ -11788,7 +11868,7 @@ static struct bpf_test tests[] = { INTERNAL | FLAG_NO_DATA, { }, { { 0, 1 } }, - .fill_helper = bpf_fill_alu_lsh_imm, + .fill_helper = bpf_fill_alu64_lsh_imm, }, { "ALU64_RSH_K: all shift values", @@ -11796,7 +11876,7 @@ static struct bpf_test tests[] = { INTERNAL | FLAG_NO_DATA, { }, { { 0, 1 } }, - .fill_helper = bpf_fill_alu_rsh_imm, + .fill_helper = bpf_fill_alu64_rsh_imm, }, { "ALU64_ARSH_K: all shift values", @@ -11804,7 +11884,7 @@ static struct bpf_test tests[] = { INTERNAL | FLAG_NO_DATA, { }, { { 0, 1 } }, - .fill_helper = bpf_fill_alu_arsh_imm, + .fill_helper = bpf_fill_alu64_arsh_imm, }, { "ALU64_LSH_X: all shift values", @@ -11812,7 +11892,7 @@ static struct bpf_test tests[] = { INTERNAL | FLAG_NO_DATA, { }, { { 0, 1 } }, - .fill_helper = bpf_fill_alu_lsh_reg, + .fill_helper = bpf_fill_alu64_lsh_reg, }, { "ALU64_RSH_X: all shift values", @@ -11820,7 +11900,7 @@ static struct bpf_test tests[] = { INTERNAL | FLAG_NO_DATA, { }, { { 0, 1 } }, - .fill_helper = bpf_fill_alu_rsh_reg, + .fill_helper = bpf_fill_alu64_rsh_reg, }, { "ALU64_ARSH_X: all shift values", @@ -11828,7 +11908,7 @@ static struct bpf_test tests[] = { INTERNAL | FLAG_NO_DATA, { }, { { 0, 1 } }, - .fill_helper = bpf_fill_alu_arsh_reg, + .fill_helper = bpf_fill_alu64_arsh_reg, }, /* Exhaustive test of ALU32 shift operations */ { @@ -11879,6 +11959,62 @@ static struct bpf_test tests[] = { { { 0, 1 } }, .fill_helper = bpf_fill_alu32_arsh_reg, }, + /* + * Exhaustive test of ALU64 shift operations when + * source and destination register are the same. + */ + { + "ALU64_LSH_X: all shift values with the same register", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_lsh_same_reg, + }, + { + "ALU64_RSH_X: all shift values with the same register", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_rsh_same_reg, + }, + { + "ALU64_ARSH_X: all shift values with the same register", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_arsh_same_reg, + }, + /* + * Exhaustive test of ALU32 shift operations when + * source and destination register are the same. + */ + { + "ALU32_LSH_X: all shift values with the same register", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_lsh_same_reg, + }, + { + "ALU32_RSH_X: all shift values with the same register", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_rsh_same_reg, + }, + { + "ALU32_ARSH_X: all shift values with the same register", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_arsh_same_reg, + }, /* ALU64 immediate magnitudes */ { "ALU64_MOV_K: all immediate value magnitudes", -- cgit v1.2.3 From 7bceeb95726b105bd4241c9635acc0836df675d4 Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Fri, 1 Oct 2021 15:03:48 +0200 Subject: bpf/tests: Add test of LDX_MEM with operand aliasing This patch adds a set of tests of BPF_LDX_MEM where both operand registers are the same register. Mainly testing 32-bit JITs that may load a 64-bit value in two 32-bit loads, and must not overwrite the address register. Signed-off-by: Johan Almbladh Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20211001130348.3670534-11-johan.almbladh@anyfinetworks.com --- lib/test_bpf.c | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index dfcbdff714b6..b9fc330fc83b 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -11133,6 +11133,64 @@ static struct bpf_test tests[] = { {}, { { 0, 2 } }, }, + /* BPF_LDX_MEM with operand aliasing */ + { + "LDX_MEM_B: operand register aliasing", + .u.insns_int = { + BPF_ST_MEM(BPF_B, R10, -8, 123), + BPF_MOV64_REG(R0, R10), + BPF_LDX_MEM(BPF_B, R0, R0, -8), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 123 } }, + .stack_depth = 8, + }, + { + "LDX_MEM_H: operand register aliasing", + .u.insns_int = { + BPF_ST_MEM(BPF_H, R10, -8, 12345), + BPF_MOV64_REG(R0, R10), + BPF_LDX_MEM(BPF_H, R0, R0, -8), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 12345 } }, + .stack_depth = 8, + }, + { + "LDX_MEM_W: operand register aliasing", + .u.insns_int = { + BPF_ST_MEM(BPF_W, R10, -8, 123456789), + BPF_MOV64_REG(R0, R10), + BPF_LDX_MEM(BPF_W, R0, R0, -8), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 123456789 } }, + .stack_depth = 8, + }, + { + "LDX_MEM_DW: operand register aliasing", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x123456789abcdefULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), + BPF_MOV64_REG(R0, R10), + BPF_LDX_MEM(BPF_DW, R0, R0, -8), + BPF_ALU64_REG(BPF_SUB, R0, R1), + BPF_MOV64_REG(R1, R0), + BPF_ALU64_IMM(BPF_RSH, R1, 32), + BPF_ALU64_REG(BPF_OR, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, /* * Register (non-)clobbering tests for the case where a JIT implements * complex ALU or ATOMIC operations via function calls. If so, the -- cgit v1.2.3 From 0eb4ef88c53f7169c44b1bd2ace5389981409a60 Mon Sep 17 00:00:00 2001 From: Johan Almbladh Date: Thu, 7 Oct 2021 16:30:06 +0200 Subject: bpf, tests: Add more LD_IMM64 tests This patch adds new tests for the two-instruction LD_IMM64. The new tests verify the operation with immediate values of different byte patterns. Mainly intended to cover JITs that want to be clever when loading 64-bit constants. Signed-off-by: Johan Almbladh Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20211007143006.634308-1-johan.almbladh@anyfinetworks.com --- lib/test_bpf.c | 120 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 117 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index b9fc330fc83b..e5b10fdefab5 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -2134,7 +2134,7 @@ static int bpf_fill_atomic32_cmpxchg_reg_pairs(struct bpf_test *self) * of the immediate value. This is often the case if the native instruction * immediate field width is narrower than 32 bits. */ -static int bpf_fill_ld_imm64(struct bpf_test *self) +static int bpf_fill_ld_imm64_magn(struct bpf_test *self) { int block = 64; /* Increase for more tests per MSB position */ int len = 3 + 8 * 63 * block * 2; @@ -2180,6 +2180,88 @@ static int bpf_fill_ld_imm64(struct bpf_test *self) return 0; } +/* + * Test the two-instruction 64-bit immediate load operation for different + * combinations of bytes. Each byte in the 64-bit word is constructed as + * (base & mask) | (rand() & ~mask), where rand() is a deterministic LCG. + * All patterns (base1, mask1) and (base2, mask2) bytes are tested. + */ +static int __bpf_fill_ld_imm64_bytes(struct bpf_test *self, + u8 base1, u8 mask1, + u8 base2, u8 mask2) +{ + struct bpf_insn *insn; + int len = 3 + 8 * BIT(8); + int pattern, index; + u32 rand = 1; + int i = 0; + + insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL); + if (!insn) + return -ENOMEM; + + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); + + for (pattern = 0; pattern < BIT(8); pattern++) { + u64 imm = 0; + + for (index = 0; index < 8; index++) { + int byte; + + if (pattern & BIT(index)) + byte = (base1 & mask1) | (rand & ~mask1); + else + byte = (base2 & mask2) | (rand & ~mask2); + imm = (imm << 8) | byte; + } + + /* Update our LCG */ + rand = rand * 1664525 + 1013904223; + + /* Perform operation */ + i += __bpf_ld_imm64(&insn[i], R1, imm); + + /* Load reference */ + insn[i++] = BPF_ALU32_IMM(BPF_MOV, R2, imm); + insn[i++] = BPF_ALU32_IMM(BPF_MOV, R3, (u32)(imm >> 32)); + insn[i++] = BPF_ALU64_IMM(BPF_LSH, R3, 32); + insn[i++] = BPF_ALU64_REG(BPF_OR, R2, R3); + + /* Check result */ + insn[i++] = BPF_JMP_REG(BPF_JEQ, R1, R2, 1); + insn[i++] = BPF_EXIT_INSN(); + } + + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); + insn[i++] = BPF_EXIT_INSN(); + + self->u.ptr.insns = insn; + self->u.ptr.len = len; + BUG_ON(i != len); + + return 0; +} + +static int bpf_fill_ld_imm64_checker(struct bpf_test *self) +{ + return __bpf_fill_ld_imm64_bytes(self, 0, 0xff, 0xff, 0xff); +} + +static int bpf_fill_ld_imm64_pos_neg(struct bpf_test *self) +{ + return __bpf_fill_ld_imm64_bytes(self, 1, 0x81, 0x80, 0x80); +} + +static int bpf_fill_ld_imm64_pos_zero(struct bpf_test *self) +{ + return __bpf_fill_ld_imm64_bytes(self, 1, 0x81, 0, 0xff); +} + +static int bpf_fill_ld_imm64_neg_zero(struct bpf_test *self) +{ + return __bpf_fill_ld_imm64_bytes(self, 0x80, 0x80, 0, 0xff); +} + /* * Exhaustive tests of JMP operations for all combinations of power-of-two * magnitudes of the operands, both for positive and negative values. The @@ -12401,14 +12483,46 @@ static struct bpf_test tests[] = { .fill_helper = bpf_fill_alu32_mod_reg, .nr_testruns = NR_PATTERN_RUNS, }, - /* LD_IMM64 immediate magnitudes */ + /* LD_IMM64 immediate magnitudes and byte patterns */ { "LD_IMM64: all immediate value magnitudes", { }, INTERNAL | FLAG_NO_DATA, { }, { { 0, 1 } }, - .fill_helper = bpf_fill_ld_imm64, + .fill_helper = bpf_fill_ld_imm64_magn, + }, + { + "LD_IMM64: checker byte patterns", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_ld_imm64_checker, + }, + { + "LD_IMM64: random positive and zero byte patterns", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_ld_imm64_pos_zero, + }, + { + "LD_IMM64: random negative and zero byte patterns", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_ld_imm64_neg_zero, + }, + { + "LD_IMM64: random positive and negative byte patterns", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_ld_imm64_pos_neg, }, /* 64-bit ATOMIC register combinations */ { -- cgit v1.2.3 From 7529cc7fbd9c02eda6851f3260416cbe198a321d Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Wed, 30 Dec 2020 11:41:52 +0200 Subject: lib: bitmap: Introduce node-aware alloc API Expose new node-aware API for bitmap allocation: bitmap_alloc_node() / bitmap_zalloc_node(). Signed-off-by: Tariq Toukan Reviewed-by: Moshe Shemesh Signed-off-by: Saeed Mahameed --- include/linux/bitmap.h | 2 ++ lib/bitmap.c | 13 +++++++++++++ 2 files changed, 15 insertions(+) (limited to 'lib') diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index 37f36dad18bd..a241dcf50f39 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -123,6 +123,8 @@ struct device; */ unsigned long *bitmap_alloc(unsigned int nbits, gfp_t flags); unsigned long *bitmap_zalloc(unsigned int nbits, gfp_t flags); +unsigned long *bitmap_alloc_node(unsigned int nbits, gfp_t flags, int node); +unsigned long *bitmap_zalloc_node(unsigned int nbits, gfp_t flags, int node); void bitmap_free(const unsigned long *bitmap); /* Managed variants of the above. */ diff --git a/lib/bitmap.c b/lib/bitmap.c index 663dd81967d4..926408883456 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -1398,6 +1398,19 @@ unsigned long *bitmap_zalloc(unsigned int nbits, gfp_t flags) } EXPORT_SYMBOL(bitmap_zalloc); +unsigned long *bitmap_alloc_node(unsigned int nbits, gfp_t flags, int node) +{ + return kmalloc_array_node(BITS_TO_LONGS(nbits), sizeof(unsigned long), + flags, node); +} +EXPORT_SYMBOL(bitmap_alloc_node); + +unsigned long *bitmap_zalloc_node(unsigned int nbits, gfp_t flags, int node) +{ + return bitmap_alloc_node(nbits, flags | __GFP_ZERO, node); +} +EXPORT_SYMBOL(bitmap_zalloc_node); + void bitmap_free(const unsigned long *bitmap) { kfree(bitmap); -- cgit v1.2.3 From b066abba3ef16a4a085d237e95da0de3f0b87713 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Thu, 28 Oct 2021 09:25:21 +0800 Subject: bpf, tests: Add module parameter test_suite to test_bpf module After commit 9298e63eafea ("bpf/tests: Add exhaustive tests of ALU operand magnitudes"), when modprobe test_bpf.ko with JIT on mips64, there exists segment fault due to the following reason: [...] ALU64_MOV_X: all register value magnitudes jited:1 Break instruction in kernel code[#1] [...] It seems that the related JIT implementations of some test cases in test_bpf() have problems. At this moment, I do not care about the segment fault while I just want to verify the test cases of tail calls. Based on the above background and motivation, add the following module parameter test_suite to the test_bpf.ko: test_suite=: only the specified test suite will be run, the string can be "test_bpf", "test_tail_calls" or "test_skb_segment". If test_suite is not specified, but test_id, test_name or test_range is specified, set 'test_bpf' as the default test suite. This is useful to only test the corresponding test suite when specifying the valid test_suite string. Any invalid test suite will result in -EINVAL being returned and no tests being run. If the test_suite is not specified or specified as empty string, it does not change the current logic, all of the test cases will be run. Here are some test results: # dmesg -c # modprobe test_bpf # dmesg | grep Summary test_bpf: Summary: 1009 PASSED, 0 FAILED, [0/997 JIT'ed] test_bpf: test_tail_calls: Summary: 8 PASSED, 0 FAILED, [0/8 JIT'ed] test_bpf: test_skb_segment: Summary: 2 PASSED, 0 FAILED # rmmod test_bpf # dmesg -c # modprobe test_bpf test_suite=test_bpf # dmesg | tail -1 test_bpf: Summary: 1009 PASSED, 0 FAILED, [0/997 JIT'ed] # rmmod test_bpf # dmesg -c # modprobe test_bpf test_suite=test_tail_calls # dmesg test_bpf: #0 Tail call leaf jited:0 21 PASS [...] test_bpf: #7 Tail call error path, index out of range jited:0 32 PASS test_bpf: test_tail_calls: Summary: 8 PASSED, 0 FAILED, [0/8 JIT'ed] # rmmod test_bpf # dmesg -c # modprobe test_bpf test_suite=test_skb_segment # dmesg test_bpf: #0 gso_with_rx_frags PASS test_bpf: #1 gso_linear_no_head_frag PASS test_bpf: test_skb_segment: Summary: 2 PASSED, 0 FAILED # rmmod test_bpf # dmesg -c # modprobe test_bpf test_id=1 # dmesg test_bpf: test_bpf: set 'test_bpf' as the default test_suite. test_bpf: #1 TXA jited:0 54 51 50 PASS test_bpf: Summary: 1 PASSED, 0 FAILED, [0/1 JIT'ed] # rmmod test_bpf # dmesg -c # modprobe test_bpf test_suite=test_bpf test_name=TXA # dmesg test_bpf: #1 TXA jited:0 54 50 51 PASS test_bpf: Summary: 1 PASSED, 0 FAILED, [0/1 JIT'ed] # rmmod test_bpf # dmesg -c # modprobe test_bpf test_suite=test_tail_calls test_range=6,7 # dmesg test_bpf: #6 Tail call error path, NULL target jited:0 41 PASS test_bpf: #7 Tail call error path, index out of range jited:0 32 PASS test_bpf: test_tail_calls: Summary: 2 PASSED, 0 FAILED, [0/2 JIT'ed] # rmmod test_bpf # dmesg -c # modprobe test_bpf test_suite=test_skb_segment test_id=1 # dmesg test_bpf: #1 gso_linear_no_head_frag PASS test_bpf: test_skb_segment: Summary: 1 PASSED, 0 FAILED By the way, the above segment fault has been fixed in the latest bpf-next tree which contains the mips64 JIT rework. Signed-off-by: Tiezhu Yang Signed-off-by: Daniel Borkmann Tested-by: Johan Almbladh Acked-by: Johan Almbladh Link: https://lore.kernel.org/bpf/1635384321-28128-1-git-send-email-yangtiezhu@loongson.cn --- lib/test_bpf.c | 212 ++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 135 insertions(+), 77 deletions(-) (limited to 'lib') diff --git a/lib/test_bpf.c b/lib/test_bpf.c index e5b10fdefab5..adae39567264 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -14316,72 +14316,9 @@ module_param_string(test_name, test_name, sizeof(test_name), 0); static int test_id = -1; module_param(test_id, int, 0); -static int test_range[2] = { 0, ARRAY_SIZE(tests) - 1 }; +static int test_range[2] = { 0, INT_MAX }; module_param_array(test_range, int, NULL, 0); -static __init int find_test_index(const char *test_name) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(tests); i++) { - if (!strcmp(tests[i].descr, test_name)) - return i; - } - return -1; -} - -static __init int prepare_bpf_tests(void) -{ - if (test_id >= 0) { - /* - * if a test_id was specified, use test_range to - * cover only that test. - */ - if (test_id >= ARRAY_SIZE(tests)) { - pr_err("test_bpf: invalid test_id specified.\n"); - return -EINVAL; - } - - test_range[0] = test_id; - test_range[1] = test_id; - } else if (*test_name) { - /* - * if a test_name was specified, find it and setup - * test_range to cover only that test. - */ - int idx = find_test_index(test_name); - - if (idx < 0) { - pr_err("test_bpf: no test named '%s' found.\n", - test_name); - return -EINVAL; - } - test_range[0] = idx; - test_range[1] = idx; - } else { - /* - * check that the supplied test_range is valid. - */ - if (test_range[0] >= ARRAY_SIZE(tests) || - test_range[1] >= ARRAY_SIZE(tests) || - test_range[0] < 0 || test_range[1] < 0) { - pr_err("test_bpf: test_range is out of bound.\n"); - return -EINVAL; - } - - if (test_range[1] < test_range[0]) { - pr_err("test_bpf: test_range is ending before it starts.\n"); - return -EINVAL; - } - } - - return 0; -} - -static __init void destroy_bpf_tests(void) -{ -} - static bool exclude_test(int test_id) { return test_id < test_range[0] || test_id > test_range[1]; @@ -14553,6 +14490,10 @@ static __init int test_skb_segment(void) for (i = 0; i < ARRAY_SIZE(skb_segment_tests); i++) { const struct skb_segment_test *test = &skb_segment_tests[i]; + cond_resched(); + if (exclude_test(i)) + continue; + pr_info("#%d %s ", i, test->descr); if (test_skb_segment_single(test)) { @@ -14934,6 +14875,8 @@ static __init int test_tail_calls(struct bpf_array *progs) int ret; cond_resched(); + if (exclude_test(i)) + continue; pr_info("#%d %s ", i, test->descr); if (!fp) { @@ -14966,29 +14909,144 @@ static __init int test_tail_calls(struct bpf_array *progs) return err_cnt ? -EINVAL : 0; } +static char test_suite[32]; +module_param_string(test_suite, test_suite, sizeof(test_suite), 0); + +static __init int find_test_index(const char *test_name) +{ + int i; + + if (!strcmp(test_suite, "test_bpf")) { + for (i = 0; i < ARRAY_SIZE(tests); i++) { + if (!strcmp(tests[i].descr, test_name)) + return i; + } + } + + if (!strcmp(test_suite, "test_tail_calls")) { + for (i = 0; i < ARRAY_SIZE(tail_call_tests); i++) { + if (!strcmp(tail_call_tests[i].descr, test_name)) + return i; + } + } + + if (!strcmp(test_suite, "test_skb_segment")) { + for (i = 0; i < ARRAY_SIZE(skb_segment_tests); i++) { + if (!strcmp(skb_segment_tests[i].descr, test_name)) + return i; + } + } + + return -1; +} + +static __init int prepare_test_range(void) +{ + int valid_range; + + if (!strcmp(test_suite, "test_bpf")) + valid_range = ARRAY_SIZE(tests); + else if (!strcmp(test_suite, "test_tail_calls")) + valid_range = ARRAY_SIZE(tail_call_tests); + else if (!strcmp(test_suite, "test_skb_segment")) + valid_range = ARRAY_SIZE(skb_segment_tests); + else + return 0; + + if (test_id >= 0) { + /* + * if a test_id was specified, use test_range to + * cover only that test. + */ + if (test_id >= valid_range) { + pr_err("test_bpf: invalid test_id specified for '%s' suite.\n", + test_suite); + return -EINVAL; + } + + test_range[0] = test_id; + test_range[1] = test_id; + } else if (*test_name) { + /* + * if a test_name was specified, find it and setup + * test_range to cover only that test. + */ + int idx = find_test_index(test_name); + + if (idx < 0) { + pr_err("test_bpf: no test named '%s' found for '%s' suite.\n", + test_name, test_suite); + return -EINVAL; + } + test_range[0] = idx; + test_range[1] = idx; + } else if (test_range[0] != 0 || test_range[1] != INT_MAX) { + /* + * check that the supplied test_range is valid. + */ + if (test_range[0] < 0 || test_range[1] >= valid_range) { + pr_err("test_bpf: test_range is out of bound for '%s' suite.\n", + test_suite); + return -EINVAL; + } + + if (test_range[1] < test_range[0]) { + pr_err("test_bpf: test_range is ending before it starts.\n"); + return -EINVAL; + } + } + + return 0; +} + static int __init test_bpf_init(void) { struct bpf_array *progs = NULL; int ret; - ret = prepare_bpf_tests(); + if (strlen(test_suite) && + strcmp(test_suite, "test_bpf") && + strcmp(test_suite, "test_tail_calls") && + strcmp(test_suite, "test_skb_segment")) { + pr_err("test_bpf: invalid test_suite '%s' specified.\n", test_suite); + return -EINVAL; + } + + /* + * if test_suite is not specified, but test_id, test_name or test_range + * is specified, set 'test_bpf' as the default test suite. + */ + if (!strlen(test_suite) && + (test_id != -1 || strlen(test_name) || + (test_range[0] != 0 || test_range[1] != INT_MAX))) { + pr_info("test_bpf: set 'test_bpf' as the default test_suite.\n"); + strscpy(test_suite, "test_bpf", sizeof(test_suite)); + } + + ret = prepare_test_range(); if (ret < 0) return ret; - ret = test_bpf(); - destroy_bpf_tests(); - if (ret) - return ret; + if (!strlen(test_suite) || !strcmp(test_suite, "test_bpf")) { + ret = test_bpf(); + if (ret) + return ret; + } - ret = prepare_tail_call_tests(&progs); - if (ret) - return ret; - ret = test_tail_calls(progs); - destroy_tail_call_tests(progs); - if (ret) - return ret; + if (!strlen(test_suite) || !strcmp(test_suite, "test_tail_calls")) { + ret = prepare_tail_call_tests(&progs); + if (ret) + return ret; + ret = test_tail_calls(progs); + destroy_tail_call_tests(progs); + if (ret) + return ret; + } - return test_skb_segment(); + if (!strlen(test_suite) || !strcmp(test_suite, "test_skb_segment")) + return test_skb_segment(); + + return 0; } static void __exit test_bpf_exit(void) -- cgit v1.2.3