summaryrefslogtreecommitdiffstats
path: root/tools/lib/bpf
diff options
context:
space:
mode:
Diffstat (limited to 'tools/lib/bpf')
-rw-r--r--tools/lib/bpf/bpf.c104
-rw-r--r--tools/lib/bpf/btf.c881
-rw-r--r--tools/lib/bpf/btf.h9
-rw-r--r--tools/lib/bpf/libbpf.c589
-rw-r--r--tools/lib/bpf/libbpf.h1
-rw-r--r--tools/lib/bpf/libbpf.map13
-rw-r--r--tools/lib/bpf/libbpf_internal.h31
-rw-r--r--tools/lib/bpf/libbpf_probes.c1
-rw-r--r--tools/lib/bpf/ringbuf.c6
-rw-r--r--tools/lib/bpf/xsk.c92
-rw-r--r--tools/lib/bpf/xsk.h22
11 files changed, 1196 insertions, 553 deletions
diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c
index d27e34133973..bba48ff4c5c0 100644
--- a/tools/lib/bpf/bpf.c
+++ b/tools/lib/bpf/bpf.c
@@ -67,11 +67,12 @@ static inline int sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr,
static inline int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size)
{
+ int retries = 5;
int fd;
do {
fd = sys_bpf(BPF_PROG_LOAD, attr, size);
- } while (fd < 0 && errno == EAGAIN);
+ } while (fd < 0 && errno == EAGAIN && retries-- > 0);
return fd;
}
@@ -214,59 +215,55 @@ alloc_zero_tailing_info(const void *orecord, __u32 cnt,
return info;
}
-int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr,
- char *log_buf, size_t log_buf_sz)
+int libbpf__bpf_prog_load(const struct bpf_prog_load_params *load_attr)
{
void *finfo = NULL, *linfo = NULL;
union bpf_attr attr;
- __u32 log_level;
int fd;
- if (!load_attr || !log_buf != !log_buf_sz)
+ if (!load_attr->log_buf != !load_attr->log_buf_sz)
return -EINVAL;
- log_level = load_attr->log_level;
- if (log_level > (4 | 2 | 1) || (log_level && !log_buf))
+ if (load_attr->log_level > (4 | 2 | 1) || (load_attr->log_level && !load_attr->log_buf))
return -EINVAL;
memset(&attr, 0, sizeof(attr));
attr.prog_type = load_attr->prog_type;
attr.expected_attach_type = load_attr->expected_attach_type;
- if (attr.prog_type == BPF_PROG_TYPE_STRUCT_OPS ||
- attr.prog_type == BPF_PROG_TYPE_LSM) {
- attr.attach_btf_id = load_attr->attach_btf_id;
- } else if (attr.prog_type == BPF_PROG_TYPE_TRACING ||
- attr.prog_type == BPF_PROG_TYPE_EXT) {
- attr.attach_btf_id = load_attr->attach_btf_id;
+
+ if (load_attr->attach_prog_fd)
attr.attach_prog_fd = load_attr->attach_prog_fd;
- } else {
- attr.prog_ifindex = load_attr->prog_ifindex;
- attr.kern_version = load_attr->kern_version;
- }
- attr.insn_cnt = (__u32)load_attr->insns_cnt;
+ else
+ attr.attach_btf_obj_fd = load_attr->attach_btf_obj_fd;
+ attr.attach_btf_id = load_attr->attach_btf_id;
+
+ attr.prog_ifindex = load_attr->prog_ifindex;
+ attr.kern_version = load_attr->kern_version;
+
+ attr.insn_cnt = (__u32)load_attr->insn_cnt;
attr.insns = ptr_to_u64(load_attr->insns);
attr.license = ptr_to_u64(load_attr->license);
- attr.log_level = log_level;
- if (log_level) {
- attr.log_buf = ptr_to_u64(log_buf);
- attr.log_size = log_buf_sz;
- } else {
- attr.log_buf = ptr_to_u64(NULL);
- attr.log_size = 0;
+ attr.log_level = load_attr->log_level;
+ if (attr.log_level) {
+ attr.log_buf = ptr_to_u64(load_attr->log_buf);
+ attr.log_size = load_attr->log_buf_sz;
}
attr.prog_btf_fd = load_attr->prog_btf_fd;
+ attr.prog_flags = load_attr->prog_flags;
+
attr.func_info_rec_size = load_attr->func_info_rec_size;
attr.func_info_cnt = load_attr->func_info_cnt;
attr.func_info = ptr_to_u64(load_attr->func_info);
+
attr.line_info_rec_size = load_attr->line_info_rec_size;
attr.line_info_cnt = load_attr->line_info_cnt;
attr.line_info = ptr_to_u64(load_attr->line_info);
+
if (load_attr->name)
memcpy(attr.prog_name, load_attr->name,
- min(strlen(load_attr->name), BPF_OBJ_NAME_LEN - 1));
- attr.prog_flags = load_attr->prog_flags;
+ min(strlen(load_attr->name), (size_t)BPF_OBJ_NAME_LEN - 1));
fd = sys_bpf_prog_load(&attr, sizeof(attr));
if (fd >= 0)
@@ -306,19 +303,19 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr,
}
fd = sys_bpf_prog_load(&attr, sizeof(attr));
-
if (fd >= 0)
goto done;
}
- if (log_level || !log_buf)
+ if (load_attr->log_level || !load_attr->log_buf)
goto done;
/* Try again with log */
- attr.log_buf = ptr_to_u64(log_buf);
- attr.log_size = log_buf_sz;
+ attr.log_buf = ptr_to_u64(load_attr->log_buf);
+ attr.log_size = load_attr->log_buf_sz;
attr.log_level = 1;
- log_buf[0] = 0;
+ load_attr->log_buf[0] = 0;
+
fd = sys_bpf_prog_load(&attr, sizeof(attr));
done:
free(finfo);
@@ -326,6 +323,49 @@ done:
return fd;
}
+int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr,
+ char *log_buf, size_t log_buf_sz)
+{
+ struct bpf_prog_load_params p = {};
+
+ if (!load_attr || !log_buf != !log_buf_sz)
+ return -EINVAL;
+
+ p.prog_type = load_attr->prog_type;
+ p.expected_attach_type = load_attr->expected_attach_type;
+ switch (p.prog_type) {
+ case BPF_PROG_TYPE_STRUCT_OPS:
+ case BPF_PROG_TYPE_LSM:
+ p.attach_btf_id = load_attr->attach_btf_id;
+ break;
+ case BPF_PROG_TYPE_TRACING:
+ case BPF_PROG_TYPE_EXT:
+ p.attach_btf_id = load_attr->attach_btf_id;
+ p.attach_prog_fd = load_attr->attach_prog_fd;
+ break;
+ default:
+ p.prog_ifindex = load_attr->prog_ifindex;
+ p.kern_version = load_attr->kern_version;
+ }
+ p.insn_cnt = load_attr->insns_cnt;
+ p.insns = load_attr->insns;
+ p.license = load_attr->license;
+ p.log_level = load_attr->log_level;
+ p.log_buf = log_buf;
+ p.log_buf_sz = log_buf_sz;
+ p.prog_btf_fd = load_attr->prog_btf_fd;
+ p.func_info_rec_size = load_attr->func_info_rec_size;
+ p.func_info_cnt = load_attr->func_info_cnt;
+ p.func_info = load_attr->func_info;
+ p.line_info_rec_size = load_attr->line_info_rec_size;
+ p.line_info_cnt = load_attr->line_info_cnt;
+ p.line_info = load_attr->line_info;
+ p.name = load_attr->name;
+ p.prog_flags = load_attr->prog_flags;
+
+ return libbpf__bpf_prog_load(&p);
+}
+
int bpf_load_program(enum bpf_prog_type type, const struct bpf_insn *insns,
size_t insns_cnt, const char *license,
__u32 kern_version, char *log_buf,
diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c
index 231b07203e3d..3c3f2bc6c652 100644
--- a/tools/lib/bpf/btf.c
+++ b/tools/lib/bpf/btf.c
@@ -78,10 +78,32 @@ struct btf {
void *types_data;
size_t types_data_cap; /* used size stored in hdr->type_len */
- /* type ID to `struct btf_type *` lookup index */
+ /* type ID to `struct btf_type *` lookup index
+ * type_offs[0] corresponds to the first non-VOID type:
+ * - for base BTF it's type [1];
+ * - for split BTF it's the first non-base BTF type.
+ */
__u32 *type_offs;
size_t type_offs_cap;
+ /* number of types in this BTF instance:
+ * - doesn't include special [0] void type;
+ * - for split BTF counts number of types added on top of base BTF.
+ */
__u32 nr_types;
+ /* if not NULL, points to the base BTF on top of which the current
+ * split BTF is based
+ */
+ struct btf *base_btf;
+ /* BTF type ID of the first type in this BTF instance:
+ * - for base BTF it's equal to 1;
+ * - for split BTF it's equal to biggest type ID of base BTF plus 1.
+ */
+ int start_id;
+ /* logical string offset of this BTF instance:
+ * - for base BTF it's equal to 0;
+ * - for split BTF it's equal to total size of base BTF's string section size.
+ */
+ int start_str_off;
void *strs_data;
size_t strs_data_cap; /* used size stored in hdr->str_len */
@@ -90,6 +112,14 @@ struct btf {
struct hashmap *strs_hash;
/* whether strings are already deduplicated */
bool strs_deduped;
+ /* extra indirection layer to make strings hashmap work with stable
+ * string offsets and ability to transparently choose between
+ * btf->strs_data or btf_dedup->strs_data as a source of strings.
+ * This is used for BTF strings dedup to transfer deduplicated strings
+ * data back to struct btf without re-building strings index.
+ */
+ void **strs_data_ptr;
+
/* BTF object FD, if loaded into kernel */
int fd;
@@ -168,7 +198,7 @@ static int btf_add_type_idx_entry(struct btf *btf, __u32 type_off)
__u32 *p;
p = btf_add_mem((void **)&btf->type_offs, &btf->type_offs_cap, sizeof(__u32),
- btf->nr_types + 1, BTF_MAX_NR_TYPES, 1);
+ btf->nr_types, BTF_MAX_NR_TYPES, 1);
if (!p)
return -ENOMEM;
@@ -215,22 +245,18 @@ static int btf_parse_hdr(struct btf *btf)
return -EINVAL;
}
- if (meta_left < hdr->type_off) {
- pr_debug("Invalid BTF type section offset:%u\n", hdr->type_off);
+ if (meta_left < hdr->str_off + hdr->str_len) {
+ pr_debug("Invalid BTF total size:%u\n", btf->raw_size);
return -EINVAL;
}
- if (meta_left < hdr->str_off) {
- pr_debug("Invalid BTF string section offset:%u\n", hdr->str_off);
+ if (hdr->type_off + hdr->type_len > hdr->str_off) {
+ pr_debug("Invalid BTF data sections layout: type data at %u + %u, strings data at %u + %u\n",
+ hdr->type_off, hdr->type_len, hdr->str_off, hdr->str_len);
return -EINVAL;
}
- if (hdr->type_off >= hdr->str_off) {
- pr_debug("BTF type section offset >= string section offset. No type?\n");
- return -EINVAL;
- }
-
- if (hdr->type_off & 0x02) {
+ if (hdr->type_off % 4) {
pr_debug("BTF type section is not aligned to 4 bytes\n");
return -EINVAL;
}
@@ -244,12 +270,16 @@ static int btf_parse_str_sec(struct btf *btf)
const char *start = btf->strs_data;
const char *end = start + btf->hdr->str_len;
- if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_STR_OFFSET ||
- start[0] || end[-1]) {
+ if (btf->base_btf && hdr->str_len == 0)
+ return 0;
+ if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_STR_OFFSET || end[-1]) {
+ pr_debug("Invalid BTF string section\n");
+ return -EINVAL;
+ }
+ if (!btf->base_btf && start[0]) {
pr_debug("Invalid BTF string section\n");
return -EINVAL;
}
-
return 0;
}
@@ -364,19 +394,9 @@ static int btf_parse_type_sec(struct btf *btf)
struct btf_header *hdr = btf->hdr;
void *next_type = btf->types_data;
void *end_type = next_type + hdr->type_len;
- int err, i = 0, type_size;
-
- /* VOID (type_id == 0) is specially handled by btf__get_type_by_id(),
- * so ensure we can never properly use its offset from index by
- * setting it to a large value
- */
- err = btf_add_type_idx_entry(btf, UINT_MAX);
- if (err)
- return err;
+ int err, type_size;
while (next_type + sizeof(struct btf_type) <= end_type) {
- i++;
-
if (btf->swapped_endian)
btf_bswap_type_base(next_type);
@@ -384,7 +404,7 @@ static int btf_parse_type_sec(struct btf *btf)
if (type_size < 0)
return type_size;
if (next_type + type_size > end_type) {
- pr_warn("BTF type [%d] is malformed\n", i);
+ pr_warn("BTF type [%d] is malformed\n", btf->start_id + btf->nr_types);
return -EINVAL;
}
@@ -409,7 +429,12 @@ static int btf_parse_type_sec(struct btf *btf)
__u32 btf__get_nr_types(const struct btf *btf)
{
- return btf->nr_types;
+ return btf->start_id + btf->nr_types - 1;
+}
+
+const struct btf *btf__base_btf(const struct btf *btf)
+{
+ return btf->base_btf;
}
/* internal helper returning non-const pointer to a type */
@@ -417,13 +442,14 @@ static struct btf_type *btf_type_by_id(struct btf *btf, __u32 type_id)
{
if (type_id == 0)
return &btf_void;
-
- return btf->types_data + btf->type_offs[type_id];
+ if (type_id < btf->start_id)
+ return btf_type_by_id(btf->base_btf, type_id);
+ return btf->types_data + btf->type_offs[type_id - btf->start_id];
}
const struct btf_type *btf__type_by_id(const struct btf *btf, __u32 type_id)
{
- if (type_id > btf->nr_types)
+ if (type_id >= btf->start_id + btf->nr_types)
return NULL;
return btf_type_by_id((struct btf *)btf, type_id);
}
@@ -432,9 +458,13 @@ static int determine_ptr_size(const struct btf *btf)
{
const struct btf_type *t;
const char *name;
- int i;
+ int i, n;
- for (i = 1; i <= btf->nr_types; i++) {
+ if (btf->base_btf && btf->base_btf->ptr_sz > 0)
+ return btf->base_btf->ptr_sz;
+
+ n = btf__get_nr_types(btf);
+ for (i = 1; i <= n; i++) {
t = btf__type_by_id(btf, i);
if (!btf_is_int(t))
continue;
@@ -649,12 +679,12 @@ int btf__resolve_type(const struct btf *btf, __u32 type_id)
__s32 btf__find_by_name(const struct btf *btf, const char *type_name)
{
- __u32 i;
+ __u32 i, nr_types = btf__get_nr_types(btf);
if (!strcmp(type_name, "void"))
return 0;
- for (i = 1; i <= btf->nr_types; i++) {
+ for (i = 1; i <= nr_types; i++) {
const struct btf_type *t = btf__type_by_id(btf, i);
const char *name = btf__name_by_offset(btf, t->name_off);
@@ -668,12 +698,12 @@ __s32 btf__find_by_name(const struct btf *btf, const char *type_name)
__s32 btf__find_by_name_kind(const struct btf *btf, const char *type_name,
__u32 kind)
{
- __u32 i;
+ __u32 i, nr_types = btf__get_nr_types(btf);
if (kind == BTF_KIND_UNKN || !strcmp(type_name, "void"))
return 0;
- for (i = 1; i <= btf->nr_types; i++) {
+ for (i = 1; i <= nr_types; i++) {
const struct btf_type *t = btf__type_by_id(btf, i);
const char *name;
@@ -717,7 +747,7 @@ void btf__free(struct btf *btf)
free(btf);
}
-struct btf *btf__new_empty(void)
+static struct btf *btf_new_empty(struct btf *base_btf)
{
struct btf *btf;
@@ -725,12 +755,21 @@ struct btf *btf__new_empty(void)
if (!btf)
return ERR_PTR(-ENOMEM);
+ btf->nr_types = 0;
+ btf->start_id = 1;
+ btf->start_str_off = 0;
btf->fd = -1;
btf->ptr_sz = sizeof(void *);
btf->swapped_endian = false;
+ if (base_btf) {
+ btf->base_btf = base_btf;
+ btf->start_id = btf__get_nr_types(base_btf) + 1;
+ btf->start_str_off = base_btf->hdr->str_len;
+ }
+
/* +1 for empty string at offset 0 */
- btf->raw_size = sizeof(struct btf_header) + 1;
+ btf->raw_size = sizeof(struct btf_header) + (base_btf ? 0 : 1);
btf->raw_data = calloc(1, btf->raw_size);
if (!btf->raw_data) {
free(btf);
@@ -744,12 +783,22 @@ struct btf *btf__new_empty(void)
btf->types_data = btf->raw_data + btf->hdr->hdr_len;
btf->strs_data = btf->raw_data + btf->hdr->hdr_len;
- btf->hdr->str_len = 1; /* empty string at offset 0 */
+ btf->hdr->str_len = base_btf ? 0 : 1; /* empty string at offset 0 */
return btf;
}
-struct btf *btf__new(const void *data, __u32 size)
+struct btf *btf__new_empty(void)
+{
+ return btf_new_empty(NULL);
+}
+
+struct btf *btf__new_empty_split(struct btf *base_btf)
+{
+ return btf_new_empty(base_btf);
+}
+
+static struct btf *btf_new(const void *data, __u32 size, struct btf *base_btf)
{
struct btf *btf;
int err;
@@ -758,6 +807,16 @@ struct btf *btf__new(const void *data, __u32 size)
if (!btf)
return ERR_PTR(-ENOMEM);
+ btf->nr_types = 0;
+ btf->start_id = 1;
+ btf->start_str_off = 0;
+
+ if (base_btf) {
+ btf->base_btf = base_btf;
+ btf->start_id = btf__get_nr_types(base_btf) + 1;
+ btf->start_str_off = base_btf->hdr->str_len;
+ }
+
btf->raw_data = malloc(size);
if (!btf->raw_data) {
err = -ENOMEM;
@@ -790,7 +849,13 @@ done:
return btf;
}
-struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext)
+struct btf *btf__new(const void *data, __u32 size)
+{
+ return btf_new(data, size, NULL);
+}
+
+static struct btf *btf_parse_elf(const char *path, struct btf *base_btf,
+ struct btf_ext **btf_ext)
{
Elf_Data *btf_data = NULL, *btf_ext_data = NULL;
int err = 0, fd = -1, idx = 0;
@@ -868,7 +933,7 @@ struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext)
err = -ENOENT;
goto done;
}
- btf = btf__new(btf_data->d_buf, btf_data->d_size);
+ btf = btf_new(btf_data->d_buf, btf_data->d_size, base_btf);
if (IS_ERR(btf))
goto done;
@@ -913,7 +978,17 @@ done:
return btf;
}
-struct btf *btf__parse_raw(const char *path)
+struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext)
+{
+ return btf_parse_elf(path, NULL, btf_ext);
+}
+
+struct btf *btf__parse_elf_split(const char *path, struct btf *base_btf)
+{
+ return btf_parse_elf(path, base_btf, NULL);
+}
+
+static struct btf *btf_parse_raw(const char *path, struct btf *base_btf)
{
struct btf *btf = NULL;
void *data = NULL;
@@ -967,7 +1042,7 @@ struct btf *btf__parse_raw(const char *path)
}
/* finally parse BTF data */
- btf = btf__new(data, sz);
+ btf = btf_new(data, sz, base_btf);
err_out:
free(data);
@@ -976,18 +1051,38 @@ err_out:
return err ? ERR_PTR(err) : btf;
}
-struct btf *btf__parse(const char *path, struct btf_ext **btf_ext)
+struct btf *btf__parse_raw(const char *path)
+{
+ return btf_parse_raw(path, NULL);
+}
+
+struct btf *btf__parse_raw_split(const char *path, struct btf *base_btf)
+{
+ return btf_parse_raw(path, base_btf);
+}
+
+static struct btf *btf_parse(const char *path, struct btf *base_btf, struct btf_ext **btf_ext)
{
struct btf *btf;
if (btf_ext)
*btf_ext = NULL;
- btf = btf__parse_raw(path);
+ btf = btf_parse_raw(path, base_btf);
if (!IS_ERR(btf) || PTR_ERR(btf) != -EPROTO)
return btf;
- return btf__parse_elf(path, btf_ext);
+ return btf_parse_elf(path, base_btf, btf_ext);
+}
+
+struct btf *btf__parse(const char *path, struct btf_ext **btf_ext)
+{
+ return btf_parse(path, NULL, btf_ext);
+}
+
+struct btf *btf__parse_split(const char *path, struct btf *base_btf)
+{
+ return btf_parse(path, base_btf, NULL);
}
static int compare_vsi_off(const void *_a, const void *_b)
@@ -1171,8 +1266,8 @@ static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endi
memcpy(p, btf->types_data, hdr->type_len);
if (swap_endian) {
- for (i = 1; i <= btf->nr_types; i++) {
- t = p + btf->type_offs[i];
+ for (i = 0; i < btf->nr_types; i++) {
+ t = p + btf->type_offs[i];
/* btf_bswap_type_rest() relies on native t->info, so
* we swap base type info after we swapped all the
* additional information
@@ -1215,8 +1310,10 @@ const void *btf__get_raw_data(const struct btf *btf_ro, __u32 *size)
const char *btf__str_by_offset(const struct btf *btf, __u32 offset)
{
- if (offset < btf->hdr->str_len)
- return btf->strs_data + offset;
+ if (offset < btf->start_str_off)
+ return btf__str_by_offset(btf->base_btf, offset);
+ else if (offset - btf->start_str_off < btf->hdr->str_len)
+ return btf->strs_data + (offset - btf->start_str_off);
else
return NULL;
}
@@ -1226,35 +1323,27 @@ const char *btf__name_by_offset(const struct btf *btf, __u32 offset)
return btf__str_by_offset(btf, offset);
}
-int btf__get_from_id(__u32 id, struct btf **btf)
+struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf)
{
- struct bpf_btf_info btf_info = { 0 };
+ struct bpf_btf_info btf_info;
__u32 len = sizeof(btf_info);
__u32 last_size;
- int btf_fd;
+ struct btf *btf;
void *ptr;
int err;
- err = 0;
- *btf = NULL;
- btf_fd = bpf_btf_get_fd_by_id(id);
- if (btf_fd < 0)
- return 0;
-
/* we won't know btf_size until we call bpf_obj_get_info_by_fd(). so
* let's start with a sane default - 4KiB here - and resize it only if
* bpf_obj_get_info_by_fd() needs a bigger buffer.
*/
- btf_info.btf_size = 4096;
- last_size = btf_info.btf_size;
+ last_size = 4096;
ptr = malloc(last_size);
- if (!ptr) {
- err = -ENOMEM;
- goto exit_free;
- }
+ if (!ptr)
+ return ERR_PTR(-ENOMEM);
- memset(ptr, 0, last_size);
+ memset(&btf_info, 0, sizeof(btf_info));
btf_info.btf = ptr_to_u64(ptr);
+ btf_info.btf_size = last_size;
err = bpf_obj_get_info_by_fd(btf_fd, &btf_info, &len);
if (!err && btf_info.btf_size > last_size) {
@@ -1263,31 +1352,48 @@ int btf__get_from_id(__u32 id, struct btf **btf)
last_size = btf_info.btf_size;
temp_ptr = realloc(ptr, last_size);
if (!temp_ptr) {
- err = -ENOMEM;
+ btf = ERR_PTR(-ENOMEM);
goto exit_free;
}
ptr = temp_ptr;
- memset(ptr, 0, last_size);
+
+ len = sizeof(btf_info);
+ memset(&btf_info, 0, sizeof(btf_info));
btf_info.btf = ptr_to_u64(ptr);
+ btf_info.btf_size = last_size;
+
err = bpf_obj_get_info_by_fd(btf_fd, &btf_info, &len);
}
if (err || btf_info.btf_size > last_size) {
- err = errno;
+ btf = err ? ERR_PTR(-errno) : ERR_PTR(-E2BIG);
goto exit_free;
}
- *btf = btf__new((__u8 *)(long)btf_info.btf, btf_info.btf_size);
- if (IS_ERR(*btf)) {
- err = PTR_ERR(*btf);
- *btf = NULL;
- }
+ btf = btf_new(ptr, btf_info.btf_size, base_btf);
exit_free:
- close(btf_fd);
free(ptr);
+ return btf;
+}
- return err;
+int btf__get_from_id(__u32 id, struct btf **btf)
+{
+ struct btf *res;
+ int btf_fd;
+
+ *btf = NULL;
+ btf_fd = bpf_btf_get_fd_by_id(id);
+ if (btf_fd < 0)
+ return -errno;
+
+ res = btf_get_from_fd(btf_fd, NULL);
+ close(btf_fd);
+ if (IS_ERR(res))
+ return PTR_ERR(res);
+
+ *btf = res;
+ return 0;
}
int btf__get_map_kv_tids(const struct btf *btf, const char *map_name,
@@ -1363,17 +1469,19 @@ int btf__get_map_kv_tids(const struct btf *btf, const char *map_name,
static size_t strs_hash_fn(const void *key, void *ctx)
{
- struct btf *btf = ctx;
- const char *str = btf->strs_data + (long)key;
+ const struct btf *btf = ctx;
+ const char *strs = *btf->strs_data_ptr;
+ const char *str = strs + (long)key;
return str_hash(str);
}
static bool strs_hash_equal_fn(const void *key1, const void *key2, void *ctx)
{
- struct btf *btf = ctx;
- const char *str1 = btf->strs_data + (long)key1;
- const char *str2 = btf->strs_data + (long)key2;
+ const struct btf *btf = ctx;
+ const char *strs = *btf->strs_data_ptr;
+ const char *str1 = strs + (long)key1;
+ const char *str2 = strs + (long)key2;
return strcmp(str1, str2) == 0;
}
@@ -1418,6 +1526,9 @@ static int btf_ensure_modifiable(struct btf *btf)
memcpy(types, btf->types_data, btf->hdr->type_len);
memcpy(strs, btf->strs_data, btf->hdr->str_len);
+ /* make hashmap below use btf->strs_data as a source of strings */
+ btf->strs_data_ptr = &btf->strs_data;
+
/* build lookup index for all strings */
hash = hashmap__new(strs_hash_fn, strs_hash_equal_fn, btf);
if (IS_ERR(hash)) {
@@ -1448,7 +1559,10 @@ static int btf_ensure_modifiable(struct btf *btf)
/* if BTF was created from scratch, all strings are guaranteed to be
* unique and deduplicated
*/
- btf->strs_deduped = btf->hdr->str_len <= 1;
+ if (btf->hdr->str_len == 0)
+ btf->strs_deduped = true;
+ if (!btf->base_btf && btf->hdr->str_len == 1)
+ btf->strs_deduped = true;
/* invalidate raw_data representation */
btf_invalidate_raw_data(btf);
@@ -1480,6 +1594,14 @@ int btf__find_str(struct btf *btf, const char *s)
long old_off, new_off, len;
void *p;
+ if (btf->base_btf) {
+ int ret;
+
+ ret = btf__find_str(btf->base_btf, s);
+ if (ret != -ENOENT)
+ return ret;
+ }
+
/* BTF needs to be in a modifiable state to build string lookup index */
if (btf_ensure_modifiable(btf))
return -ENOMEM;
@@ -1494,7 +1616,7 @@ int btf__find_str(struct btf *btf, const char *s)
memcpy(p, s, len);
if (hashmap__find(btf->strs_hash, (void *)new_off, (void **)&old_off))
- return old_off;
+ return btf->start_str_off + old_off;
return -ENOENT;
}
@@ -1510,6 +1632,14 @@ int btf__add_str(struct btf *btf, const char *s)
void *p;
int err;
+ if (btf->base_btf) {
+ int ret;
+
+ ret = btf__find_str(btf->base_btf, s);
+ if (ret != -ENOENT)
+ return ret;
+ }
+
if (btf_ensure_modifiable(btf))
return -ENOMEM;
@@ -1536,12 +1666,12 @@ int btf__add_str(struct btf *btf, const char *s)
err = hashmap__insert(btf->strs_hash, (void *)new_off, (void *)new_off,
HASHMAP_ADD, (const void **)&old_off, NULL);
if (err == -EEXIST)
- return old_off; /* duplicated string, return existing offset */
+ return btf->start_str_off + old_off; /* duplicated string, return existing offset */
if (err)
return err;
btf->hdr->str_len += len; /* new unique string, adjust data length */
- return new_off;
+ return btf->start_str_off + new_off;
}
static void *btf_add_type_mem(struct btf *btf, size_t add_sz)
@@ -1560,6 +1690,20 @@ static void btf_type_inc_vlen(struct btf_type *t)
t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, btf_kflag(t));
}
+static int btf_commit_type(struct btf *btf, int data_sz)
+{
+ int err;
+
+ err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
+ if (err)
+ return err;
+
+ btf->hdr->type_len += data_sz;
+ btf->hdr->str_off += data_sz;
+ btf->nr_types++;
+ return btf->start_id + btf->nr_types - 1;
+}
+
/*
* Append new BTF_KIND_INT type with:
* - *name* - non-empty, non-NULL type name;
@@ -1572,7 +1716,7 @@ static void btf_type_inc_vlen(struct btf_type *t)
int btf__add_int(struct btf *btf, const char *name, size_t byte_sz, int encoding)
{
struct btf_type *t;
- int sz, err, name_off;
+ int sz, name_off;
/* non-empty name */
if (!name || !name[0])
@@ -1606,14 +1750,7 @@ int btf__add_int(struct btf *btf, const char *name, size_t byte_sz, int encoding
/* set INT info, we don't allow setting legacy bit offset/size */
*(__u32 *)(t + 1) = (encoding << 24) | (byte_sz * 8);
- err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
- if (err)
- return err;
-
- btf->hdr->type_len += sz;
- btf->hdr->str_off += sz;
- btf->nr_types++;
- return btf->nr_types;
+ return btf_commit_type(btf, sz);
}
/* it's completely legal to append BTF types with type IDs pointing forward to
@@ -1631,7 +1768,7 @@ static int validate_type_id(int id)
static int btf_add_ref_kind(struct btf *btf, int kind, const char *name, int ref_type_id)
{
struct btf_type *t;
- int sz, name_off = 0, err;
+ int sz, name_off = 0;
if (validate_type_id(ref_type_id))
return -EINVAL;
@@ -1654,14 +1791,7 @@ static int btf_add_ref_kind(struct btf *btf, int kind, const char *name, int ref
t->info = btf_type_info(kind, 0, 0);
t->type = ref_type_id;
- err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
- if (err)
- return err;
-
- btf->hdr->type_len += sz;
- btf->hdr->str_off += sz;
- btf->nr_types++;
- return btf->nr_types;
+ return btf_commit_type(btf, sz);
}
/*
@@ -1689,7 +1819,7 @@ int btf__add_array(struct btf *btf, int index_type_id, int elem_type_id, __u32 n
{
struct btf_type *t;
struct btf_array *a;
- int sz, err;
+ int sz;
if (validate_type_id(index_type_id) || validate_type_id(elem_type_id))
return -EINVAL;
@@ -1711,21 +1841,14 @@ int btf__add_array(struct btf *btf, int index_type_id, int elem_type_id, __u32 n
a->index_type = index_type_id;
a->nelems = nr_elems;
- err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
- if (err)
- return err;
-
- btf->hdr->type_len += sz;
- btf->hdr->str_off += sz;
- btf->nr_types++;
- return btf->nr_types;
+ return btf_commit_type(btf, sz);
}
/* generic STRUCT/UNION append function */
static int btf_add_composite(struct btf *btf, int kind, const char *name, __u32 bytes_sz)
{
struct btf_type *t;
- int sz, err, name_off = 0;
+ int sz, name_off = 0;
if (btf_ensure_modifiable(btf))
return -ENOMEM;
@@ -1748,14 +1871,7 @@ static int btf_add_composite(struct btf *btf, int kind, const char *name, __u32
t->info = btf_type_info(kind, 0, 0);
t->size = bytes_sz;
- err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
- if (err)
- return err;
-
- btf->hdr->type_len += sz;
- btf->hdr->str_off += sz;
- btf->nr_types++;
- return btf->nr_types;
+ return btf_commit_type(btf, sz);
}
/*
@@ -1793,6 +1909,11 @@ int btf__add_union(struct btf *btf, const char *name, __u32 byte_sz)
return btf_add_composite(btf, BTF_KIND_UNION, name, byte_sz);
}
+static struct btf_type *btf_last_type(struct btf *btf)
+{
+ return btf_type_by_id(btf, btf__get_nr_types(btf));
+}
+
/*
* Append new field for the current STRUCT/UNION type with:
* - *name* - name of the field, can be NULL or empty for anonymous field;
@@ -1814,7 +1935,7 @@ int btf__add_field(struct btf *btf, const char *name, int type_id,
/* last type should be union/struct */
if (btf->nr_types == 0)
return -EINVAL;
- t = btf_type_by_id(btf, btf->nr_types);
+ t = btf_last_type(btf);
if (!btf_is_composite(t))
return -EINVAL;
@@ -1849,7 +1970,7 @@ int btf__add_field(struct btf *btf, const char *name, int type_id,
m->offset = bit_offset | (bit_size << 24);
/* btf_add_type_mem can invalidate t pointer */
- t = btf_type_by_id(btf, btf->nr_types);
+ t = btf_last_type(btf);
/* update parent type's vlen and kflag */
t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, is_bitfield || btf_kflag(t));
@@ -1874,7 +1995,7 @@ int btf__add_field(struct btf *btf, const char *name, int type_id,
int btf__add_enum(struct btf *btf, const char *name, __u32 byte_sz)
{
struct btf_type *t;
- int sz, err, name_off = 0;
+ int sz, name_off = 0;
/* byte_sz must be power of 2 */
if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 8)
@@ -1899,14 +2020,7 @@ int btf__add_enum(struct btf *btf, const char *name, __u32 byte_sz)
t->info = btf_type_info(BTF_KIND_ENUM, 0, 0);
t->size = byte_sz;
- err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
- if (err)
- return err;
-
- btf->hdr->type_len += sz;
- btf->hdr->str_off += sz;
- btf->nr_types++;
- return btf->nr_types;
+ return btf_commit_type(btf, sz);
}
/*
@@ -1926,7 +2040,7 @@ int btf__add_enum_value(struct btf *btf, const char *name, __s64 value)
/* last type should be BTF_KIND_ENUM */
if (btf->nr_types == 0)
return -EINVAL;
- t = btf_type_by_id(btf, btf->nr_types);
+ t = btf_last_type(btf);
if (!btf_is_enum(t))
return -EINVAL;
@@ -1953,7 +2067,7 @@ int btf__add_enum_value(struct btf *btf, const char *name, __s64 value)
v->val = value;
/* update parent type's vlen */
- t = btf_type_by_id(btf, btf->nr_types);
+ t = btf_last_type(btf);
btf_type_inc_vlen(t);
btf->hdr->type_len += sz;
@@ -2093,7 +2207,7 @@ int btf__add_func(struct btf *btf, const char *name,
int btf__add_func_proto(struct btf *btf, int ret_type_id)
{
struct btf_type *t;
- int sz, err;
+ int sz;
if (validate_type_id(ret_type_id))
return -EINVAL;
@@ -2113,14 +2227,7 @@ int btf__add_func_proto(struct btf *btf, int ret_type_id)
t->info = btf_type_info(BTF_KIND_FUNC_PROTO, 0, 0);
t->type = ret_type_id;
- err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
- if (err)
- return err;
-
- btf->hdr->type_len += sz;
- btf->hdr->str_off += sz;
- btf->nr_types++;
- return btf->nr_types;
+ return btf_commit_type(btf, sz);
}
/*
@@ -2143,7 +2250,7 @@ int btf__add_func_param(struct btf *btf, const char *name, int type_id)
/* last type should be BTF_KIND_FUNC_PROTO */
if (btf->nr_types == 0)
return -EINVAL;
- t = btf_type_by_id(btf, btf->nr_types);
+ t = btf_last_type(btf);
if (!btf_is_func_proto(t))
return -EINVAL;
@@ -2166,7 +2273,7 @@ int btf__add_func_param(struct btf *btf, const char *name, int type_id)
p->type = type_id;
/* update parent type's vlen */
- t = btf_type_by_id(btf, btf->nr_types);
+ t = btf_last_type(btf);
btf_type_inc_vlen(t);
btf->hdr->type_len += sz;
@@ -2188,7 +2295,7 @@ int btf__add_var(struct btf *btf, const char *name, int linkage, int type_id)
{
struct btf_type *t;
struct btf_var *v;
- int sz, err, name_off;
+ int sz, name_off;
/* non-empty name */
if (!name || !name[0])
@@ -2219,14 +2326,7 @@ int btf__add_var(struct btf *btf, const char *name, int linkage, int type_id)
v = btf_var(t);
v->linkage = linkage;
- err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
- if (err)
- return err;
-
- btf->hdr->type_len += sz;
- btf->hdr->str_off += sz;
- btf->nr_types++;
- return btf->nr_types;
+ return btf_commit_type(btf, sz);
}
/*
@@ -2244,7 +2344,7 @@ int btf__add_var(struct btf *btf, const char *name, int linkage, int type_id)
int btf__add_datasec(struct btf *btf, const char *name, __u32 byte_sz)
{
struct btf_type *t;
- int sz, err, name_off;
+ int sz, name_off;
/* non-empty name */
if (!name || !name[0])
@@ -2267,14 +2367,7 @@ int btf__add_datasec(struct btf *btf, const char *name, __u32 byte_sz)
t->info = btf_type_info(BTF_KIND_DATASEC, 0, 0);
t->size = byte_sz;
- err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
- if (err)
- return err;
-
- btf->hdr->type_len += sz;
- btf->hdr->str_off += sz;
- btf->nr_types++;
- return btf->nr_types;
+ return btf_commit_type(btf, sz);
}
/*
@@ -2296,7 +2389,7 @@ int btf__add_datasec_var_info(struct btf *btf, int var_type_id, __u32 offset, __
/* last type should be BTF_KIND_DATASEC */
if (btf->nr_types == 0)
return -EINVAL;
- t = btf_type_by_id(btf, btf->nr_types);
+ t = btf_last_type(btf);
if (!btf_is_datasec(t))
return -EINVAL;
@@ -2317,7 +2410,7 @@ int btf__add_datasec_var_info(struct btf *btf, int var_type_id, __u32 offset, __
v->size = byte_sz;
/* update parent type's vlen */
- t = btf_type_by_id(btf, btf->nr_types);
+ t = btf_last_type(btf);
btf_type_inc_vlen(t);
btf->hdr->type_len += sz;
@@ -2639,6 +2732,7 @@ struct btf_dedup;
static struct btf_dedup *btf_dedup_new(struct btf *btf, struct btf_ext *btf_ext,
const struct btf_dedup_opts *opts);
static void btf_dedup_free(struct btf_dedup *d);
+static int btf_dedup_prep(struct btf_dedup *d);
static int btf_dedup_strings(struct btf_dedup *d);
static int btf_dedup_prim_types(struct btf_dedup *d);
static int btf_dedup_struct_types(struct btf_dedup *d);
@@ -2797,6 +2891,11 @@ int btf__dedup(struct btf *btf, struct btf_ext *btf_ext,
if (btf_ensure_modifiable(btf))
return -ENOMEM;
+ err = btf_dedup_prep(d);
+ if (err) {
+ pr_debug("btf_dedup_prep failed:%d\n", err);
+ goto done;
+ }
err = btf_dedup_strings(d);
if (err < 0) {
pr_debug("btf_dedup_strings failed:%d\n", err);
@@ -2859,21 +2958,20 @@ struct btf_dedup {
__u32 *hypot_list;
size_t hypot_cnt;
size_t hypot_cap;
+ /* Whether hypothetical mapping, if successful, would need to adjust
+ * already canonicalized types (due to a new forward declaration to
+ * concrete type resolution). In such case, during split BTF dedup
+ * candidate type would still be considered as different, because base
+ * BTF is considered to be immutable.
+ */
+ bool hypot_adjust_canon;
/* Various option modifying behavior of algorithm */
struct btf_dedup_opts opts;
-};
-
-struct btf_str_ptr {
- const char *str;
- __u32 new_off;
- bool used;
-};
-
-struct btf_str_ptrs {
- struct btf_str_ptr *ptrs;
- const char *data;
- __u32 cnt;
- __u32 cap;
+ /* temporary strings deduplication state */
+ void *strs_data;
+ size_t strs_cap;
+ size_t strs_len;
+ struct hashmap* strs_hash;
};
static long hash_combine(long h, long value)
@@ -2914,6 +3012,7 @@ static void btf_dedup_clear_hypot_map(struct btf_dedup *d)
for (i = 0; i < d->hypot_cnt; i++)
d->hypot_map[d->hypot_list[i]] = BTF_UNPROCESSED_ID;
d->hypot_cnt = 0;
+ d->hypot_adjust_canon = false;
}
static void btf_dedup_free(struct btf_dedup *d)
@@ -2953,7 +3052,7 @@ static struct btf_dedup *btf_dedup_new(struct btf *btf, struct btf_ext *btf_ext,
{
struct btf_dedup *d = calloc(1, sizeof(struct btf_dedup));
hashmap_hash_fn hash_fn = btf_dedup_identity_hash_fn;
- int i, err = 0;
+ int i, err = 0, type_cnt;
if (!d)
return ERR_PTR(-ENOMEM);
@@ -2973,14 +3072,15 @@ static struct btf_dedup *btf_dedup_new(struct btf *btf, struct btf_ext *btf_ext,
goto done;
}
- d->map = malloc(sizeof(__u32) * (1 + btf->nr_types));
+ type_cnt = btf__get_nr_types(btf) + 1;
+ d->map = malloc(sizeof(__u32) * type_cnt);
if (!d->map) {
err = -ENOMEM;
goto done;
}
/* special BTF "void" type is made canonical immediately */
d->map[0] = 0;
- for (i = 1; i <= btf->nr_types; i++) {
+ for (i = 1; i < type_cnt; i++) {
struct btf_type *t = btf_type_by_id(d->btf, i);
/* VAR and DATASEC are never deduped and are self-canonical */
@@ -2990,12 +3090,12 @@ static struct btf_dedup *btf_dedup_new(struct btf *btf, struct btf_ext *btf_ext,
d->map[i] = BTF_UNPROCESSED_ID;
}
- d->hypot_map = malloc(sizeof(__u32) * (1 + btf->nr_types));
+ d->hypot_map = malloc(sizeof(__u32) * type_cnt);
if (!d->hypot_map) {
err = -ENOMEM;
goto done;
}
- for (i = 0; i <= btf->nr_types; i++)
+ for (i = 0; i < type_cnt; i++)
d->hypot_map[i] = BTF_UNPROCESSED_ID;
done:
@@ -3019,8 +3119,8 @@ static int btf_for_each_str_off(struct btf_dedup *d, str_off_fn_t fn, void *ctx)
int i, j, r, rec_size;
struct btf_type *t;
- for (i = 1; i <= d->btf->nr_types; i++) {
- t = btf_type_by_id(d->btf, i);
+ for (i = 0; i < d->btf->nr_types; i++) {
+ t = btf_type_by_id(d->btf, d->btf->start_id + i);
r = fn(&t->name_off, ctx);
if (r)
return r;
@@ -3100,64 +3200,53 @@ static int btf_for_each_str_off(struct btf_dedup *d, str_off_fn_t fn, void *ctx)
return 0;
}
-static int str_sort_by_content(const void *a1, const void *a2)
+static int strs_dedup_remap_str_off(__u32 *str_off_ptr, void *ctx)
{
- const struct btf_str_ptr *p1 = a1;
- const struct btf_str_ptr *p2 = a2;
-
- return strcmp(p1->str, p2->str);
-}
-
-static int str_sort_by_offset(const void *a1, const void *a2)
-{
- const struct btf_str_ptr *p1 = a1;
- const struct btf_str_ptr *p2 = a2;
-
- if (p1->str != p2->str)
- return p1->str < p2->str ? -1 : 1;
- return 0;
-}
-
-static int btf_dedup_str_ptr_cmp(const void *str_ptr, const void *pelem)
-{
- const struct btf_str_ptr *p = pelem;
-
- if (str_ptr != p->str)
- return (const char *)str_ptr < p->str ? -1 : 1;
- return 0;
-}
-
-static int btf_str_mark_as_used(__u32 *str_off_ptr, void *ctx)
-{
- struct btf_str_ptrs *strs;
- struct btf_str_ptr *s;
+ struct btf_dedup *d = ctx;
+ __u32 str_off = *str_off_ptr;
+ long old_off, new_off, len;
+ const char *s;
+ void *p;
+ int err;
- if (*str_off_ptr == 0)
+ /* don't touch empty string or string in main BTF */
+ if (str_off == 0 || str_off < d->btf->start_str_off)
return 0;
- strs = ctx;
- s = bsearch(strs->data + *str_off_ptr, strs->ptrs, strs->cnt,
- sizeof(struct btf_str_ptr), btf_dedup_str_ptr_cmp);
- if (!s)
- return -EINVAL;
- s->used = true;
- return 0;
-}
+ s = btf__str_by_offset(d->btf, str_off);
+ if (d->btf->base_btf) {
+ err = btf__find_str(d->btf->base_btf, s);
+ if (err >= 0) {
+ *str_off_ptr = err;
+ return 0;
+ }
+ if (err != -ENOENT)
+ return err;
+ }
-static int btf_str_remap_offset(__u32 *str_off_ptr, void *ctx)
-{
- struct btf_str_ptrs *strs;
- struct btf_str_ptr *s;
+ len = strlen(s) + 1;
- if (*str_off_ptr == 0)
- return 0;
+ new_off = d->strs_len;
+ p = btf_add_mem(&d->strs_data, &d->strs_cap, 1, new_off, BTF_MAX_STR_OFFSET, len);
+ if (!p)
+ return -ENOMEM;
- strs = ctx;
- s = bsearch(strs->data + *str_off_ptr, strs->ptrs, strs->cnt,
- sizeof(struct btf_str_ptr), btf_dedup_str_ptr_cmp);
- if (!s)
- return -EINVAL;
- *str_off_ptr = s->new_off;
+ memcpy(p, s, len);
+
+ /* Now attempt to add the string, but only if the string with the same
+ * contents doesn't exist already (HASHMAP_ADD strategy). If such
+ * string exists, we'll get its offset in old_off (that's old_key).
+ */
+ err = hashmap__insert(d->strs_hash, (void *)new_off, (void *)new_off,
+ HASHMAP_ADD, (const void **)&old_off, NULL);
+ if (err == -EEXIST) {
+ *str_off_ptr = d->btf->start_str_off + old_off;
+ } else if (err) {
+ return err;
+ } else {
+ *str_off_ptr = d->btf->start_str_off + new_off;
+ d->strs_len += len;
+ }
return 0;
}
@@ -3174,118 +3263,71 @@ static int btf_str_remap_offset(__u32 *str_off_ptr, void *ctx)
*/
static int btf_dedup_strings(struct btf_dedup *d)
{
- char *start = d->btf->strs_data;
- char *end = start + d->btf->hdr->str_len;
- char *p = start, *tmp_strs = NULL;
- struct btf_str_ptrs strs = {
- .cnt = 0,
- .cap = 0,
- .ptrs = NULL,
- .data = start,
- };
- int i, j, err = 0, grp_idx;
- bool grp_used;
+ char *s;
+ int err;
if (d->btf->strs_deduped)
return 0;
- /* build index of all strings */
- while (p < end) {
- if (strs.cnt + 1 > strs.cap) {
- struct btf_str_ptr *new_ptrs;
-
- strs.cap += max(strs.cnt / 2, 16U);
- new_ptrs = libbpf_reallocarray(strs.ptrs, strs.cap, sizeof(strs.ptrs[0]));
- if (!new_ptrs) {
- err = -ENOMEM;
- goto done;
- }
- strs.ptrs = new_ptrs;
- }
-
- strs.ptrs[strs.cnt].str = p;
- strs.ptrs[strs.cnt].used = false;
-
- p += strlen(p) + 1;
- strs.cnt++;
- }
+ /* temporarily switch to use btf_dedup's strs_data for strings for hash
+ * functions; later we'll just transfer hashmap to struct btf as is,
+ * along the strs_data
+ */
+ d->btf->strs_data_ptr = &d->strs_data;
- /* temporary storage for deduplicated strings */
- tmp_strs = malloc(d->btf->hdr->str_len);
- if (!tmp_strs) {
- err = -ENOMEM;
- goto done;
+ d->strs_hash = hashmap__new(strs_hash_fn, strs_hash_equal_fn, d->btf);
+ if (IS_ERR(d->strs_hash)) {
+ err = PTR_ERR(d->strs_hash);
+ d->strs_hash = NULL;
+ goto err_out;
}
- /* mark all used strings */
- strs.ptrs[0].used = true;
- err = btf_for_each_str_off(d, btf_str_mark_as_used, &strs);
- if (err)
- goto done;
-
- /* sort strings by context, so that we can identify duplicates */
- qsort(strs.ptrs, strs.cnt, sizeof(strs.ptrs[0]), str_sort_by_content);
-
- /*
- * iterate groups of equal strings and if any instance in a group was
- * referenced, emit single instance and remember new offset
- */
- p = tmp_strs;
- grp_idx = 0;
- grp_used = strs.ptrs[0].used;
- /* iterate past end to avoid code duplication after loop */
- for (i = 1; i <= strs.cnt; i++) {
- /*
- * when i == strs.cnt, we want to skip string comparison and go
- * straight to handling last group of strings (otherwise we'd
- * need to handle last group after the loop w/ duplicated code)
- */
- if (i < strs.cnt &&
- !strcmp(strs.ptrs[i].str, strs.ptrs[grp_idx].str)) {
- grp_used = grp_used || strs.ptrs[i].used;
- continue;
- }
+ if (!d->btf->base_btf) {
+ s = btf_add_mem(&d->strs_data, &d->strs_cap, 1, d->strs_len, BTF_MAX_STR_OFFSET, 1);
+ if (!s)
+ return -ENOMEM;
+ /* initial empty string */
+ s[0] = 0;
+ d->strs_len = 1;
- /*
- * this check would have been required after the loop to handle
- * last group of strings, but due to <= condition in a loop
- * we avoid that duplication
+ /* insert empty string; we won't be looking it up during strings
+ * dedup, but it's good to have it for generic BTF string lookups
*/
- if (grp_used) {
- int new_off = p - tmp_strs;
- __u32 len = strlen(strs.ptrs[grp_idx].str);
-
- memmove(p, strs.ptrs[grp_idx].str, len + 1);
- for (j = grp_idx; j < i; j++)
- strs.ptrs[j].new_off = new_off;
- p += len + 1;
- }
-
- if (i < strs.cnt) {
- grp_idx = i;
- grp_used = strs.ptrs[i].used;
- }
+ err = hashmap__insert(d->strs_hash, (void *)0, (void *)0,
+ HASHMAP_ADD, NULL, NULL);
+ if (err)
+ goto err_out;
}
- /* replace original strings with deduped ones */
- d->btf->hdr->str_len = p - tmp_strs;
- memmove(start, tmp_strs, d->btf->hdr->str_len);
- end = start + d->btf->hdr->str_len;
-
- /* restore original order for further binary search lookups */
- qsort(strs.ptrs, strs.cnt, sizeof(strs.ptrs[0]), str_sort_by_offset);
-
/* remap string offsets */
- err = btf_for_each_str_off(d, btf_str_remap_offset, &strs);
+ err = btf_for_each_str_off(d, strs_dedup_remap_str_off, d);
if (err)
- goto done;
+ goto err_out;
- d->btf->hdr->str_len = end - start;
+ /* replace BTF string data and hash with deduped ones */
+ free(d->btf->strs_data);
+ hashmap__free(d->btf->strs_hash);
+ d->btf->strs_data = d->strs_data;
+ d->btf->strs_data_cap = d->strs_cap;
+ d->btf->hdr->str_len = d->strs_len;
+ d->btf->strs_hash = d->strs_hash;
+ /* now point strs_data_ptr back to btf->strs_data */
+ d->btf->strs_data_ptr = &d->btf->strs_data;
+
+ d->strs_data = d->strs_hash = NULL;
+ d->strs_len = d->strs_cap = 0;
d->btf->strs_deduped = true;
+ return 0;
+
+err_out:
+ free(d->strs_data);
+ hashmap__free(d->strs_hash);
+ d->strs_data = d->strs_hash = NULL;
+ d->strs_len = d->strs_cap = 0;
+
+ /* restore strings pointer for existing d->btf->strs_hash back */
+ d->btf->strs_data_ptr = &d->strs_data;
-done:
- free(tmp_strs);
- free(strs.ptrs);
return err;
}
@@ -3550,6 +3592,66 @@ static bool btf_compat_fnproto(struct btf_type *t1, struct btf_type *t2)
return true;
}
+/* Prepare split BTF for deduplication by calculating hashes of base BTF's
+ * types and initializing the rest of the state (canonical type mapping) for
+ * the fixed base BTF part.
+ */
+static int btf_dedup_prep(struct btf_dedup *d)
+{
+ struct btf_type *t;
+ int type_id;
+ long h;
+
+ if (!d->btf->base_btf)
+ return 0;
+
+ for (type_id = 1; type_id < d->btf->start_id; type_id++) {
+ t = btf_type_by_id(d->btf, type_id);
+
+ /* all base BTF types are self-canonical by definition */
+ d->map[type_id] = type_id;
+
+ switch (btf_kind(t)) {
+ case BTF_KIND_VAR:
+ case BTF_KIND_DATASEC:
+ /* VAR and DATASEC are never hash/deduplicated */
+ continue;
+ case BTF_KIND_CONST:
+ case BTF_KIND_VOLATILE:
+ case BTF_KIND_RESTRICT:
+ case BTF_KIND_PTR:
+ case BTF_KIND_FWD:
+ case BTF_KIND_TYPEDEF:
+ case BTF_KIND_FUNC:
+ h = btf_hash_common(t);
+ break;
+ case BTF_KIND_INT:
+ h = btf_hash_int(t);
+ break;
+ case BTF_KIND_ENUM:
+ h = btf_hash_enum(t);
+ break;
+ case BTF_KIND_STRUCT:
+ case BTF_KIND_UNION:
+ h = btf_hash_struct(t);
+ break;
+ case BTF_KIND_ARRAY:
+ h = btf_hash_array(t);
+ break;
+ case BTF_KIND_FUNC_PROTO:
+ h = btf_hash_fnproto(t);
+ break;
+ default:
+ pr_debug("unknown kind %d for type [%d]\n", btf_kind(t), type_id);
+ return -EINVAL;
+ }
+ if (btf_dedup_table_add(d, h, type_id))
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
/*
* Deduplicate primitive types, that can't reference other types, by calculating
* their type signature hash and comparing them with any possible canonical
@@ -3643,8 +3745,8 @@ static int btf_dedup_prim_types(struct btf_dedup *d)
{
int i, err;
- for (i = 1; i <= d->btf->nr_types; i++) {
- err = btf_dedup_prim_type(d, i);
+ for (i = 0; i < d->btf->nr_types; i++) {
+ err = btf_dedup_prim_type(d, d->btf->start_id + i);
if (err)
return err;
}
@@ -3697,6 +3799,19 @@ static inline __u16 btf_fwd_kind(struct btf_type *t)
return btf_kflag(t) ? BTF_KIND_UNION : BTF_KIND_STRUCT;
}
+/* Check if given two types are identical ARRAY definitions */
+static int btf_dedup_identical_arrays(struct btf_dedup *d, __u32 id1, __u32 id2)
+{
+ struct btf_type *t1, *t2;
+
+ t1 = btf_type_by_id(d->btf, id1);
+ t2 = btf_type_by_id(d->btf, id2);
+ if (!btf_is_array(t1) || !btf_is_array(t2))
+ return 0;
+
+ return btf_equal_array(t1, t2);
+}
+
/*
* Check equivalence of BTF type graph formed by candidate struct/union (we'll
* call it "candidate graph" in this description for brevity) to a type graph
@@ -3807,8 +3922,18 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id,
canon_id = resolve_fwd_id(d, canon_id);
hypot_type_id = d->hypot_map[canon_id];
- if (hypot_type_id <= BTF_MAX_NR_TYPES)
- return hypot_type_id == cand_id;
+ if (hypot_type_id <= BTF_MAX_NR_TYPES) {
+ /* In some cases compiler will generate different DWARF types
+ * for *identical* array type definitions and use them for
+ * different fields within the *same* struct. This breaks type
+ * equivalence check, which makes an assumption that candidate
+ * types sub-graph has a consistent and deduped-by-compiler
+ * types within a single CU. So work around that by explicitly
+ * allowing identical array types here.
+ */
+ return hypot_type_id == cand_id ||
+ btf_dedup_identical_arrays(d, hypot_type_id, cand_id);
+ }
if (btf_dedup_hypot_map_add(d, canon_id, cand_id))
return -ENOMEM;
@@ -3834,6 +3959,9 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id,
} else {
real_kind = cand_kind;
fwd_kind = btf_fwd_kind(canon_type);
+ /* we'd need to resolve base FWD to STRUCT/UNION */
+ if (fwd_kind == real_kind && canon_id < d->btf->start_id)
+ d->hypot_adjust_canon = true;
}
return fwd_kind == real_kind;
}
@@ -3871,8 +3999,7 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id,
return 0;
cand_arr = btf_array(cand_type);
canon_arr = btf_array(canon_type);
- eq = btf_dedup_is_equiv(d,
- cand_arr->index_type, canon_arr->index_type);
+ eq = btf_dedup_is_equiv(d, cand_arr->index_type, canon_arr->index_type);
if (eq <= 0)
return eq;
return btf_dedup_is_equiv(d, cand_arr->type, canon_arr->type);
@@ -3955,16 +4082,16 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id,
*/
static void btf_dedup_merge_hypot_map(struct btf_dedup *d)
{
- __u32 cand_type_id, targ_type_id;
+ __u32 canon_type_id, targ_type_id;
__u16 t_kind, c_kind;
__u32 t_id, c_id;
int i;
for (i = 0; i < d->hypot_cnt; i++) {
- cand_type_id = d->hypot_list[i];
- targ_type_id = d->hypot_map[cand_type_id];
+ canon_type_id = d->hypot_list[i];
+ targ_type_id = d->hypot_map[canon_type_id];
t_id = resolve_type_id(d, targ_type_id);
- c_id = resolve_type_id(d, cand_type_id);
+ c_id = resolve_type_id(d, canon_type_id);
t_kind = btf_kind(btf__type_by_id(d->btf, t_id));
c_kind = btf_kind(btf__type_by_id(d->btf, c_id));
/*
@@ -3979,9 +4106,26 @@ static void btf_dedup_merge_hypot_map(struct btf_dedup *d)
* stability is not a requirement for STRUCT/UNION equivalence
* checks, though.
*/
+
+ /* if it's the split BTF case, we still need to point base FWD
+ * to STRUCT/UNION in a split BTF, because FWDs from split BTF
+ * will be resolved against base FWD. If we don't point base
+ * canonical FWD to the resolved STRUCT/UNION, then all the
+ * FWDs in split BTF won't be correctly resolved to a proper
+ * STRUCT/UNION.
+ */
if (t_kind != BTF_KIND_FWD && c_kind == BTF_KIND_FWD)
d->map[c_id] = t_id;
- else if (t_kind == BTF_KIND_FWD && c_kind != BTF_KIND_FWD)
+
+ /* if graph equivalence determined that we'd need to adjust
+ * base canonical types, then we need to only point base FWDs
+ * to STRUCTs/UNIONs and do no more modifications. For all
+ * other purposes the type graphs were not equivalent.
+ */
+ if (d->hypot_adjust_canon)
+ continue;
+
+ if (t_kind == BTF_KIND_FWD && c_kind != BTF_KIND_FWD)
d->map[t_id] = c_id;
if ((t_kind == BTF_KIND_STRUCT || t_kind == BTF_KIND_UNION) &&
@@ -4065,8 +4209,10 @@ static int btf_dedup_struct_type(struct btf_dedup *d, __u32 type_id)
return eq;
if (!eq)
continue;
- new_id = cand_id;
btf_dedup_merge_hypot_map(d);
+ if (d->hypot_adjust_canon) /* not really equivalent */
+ continue;
+ new_id = cand_id;
break;
}
@@ -4081,8 +4227,8 @@ static int btf_dedup_struct_types(struct btf_dedup *d)
{
int i, err;
- for (i = 1; i <= d->btf->nr_types; i++) {
- err = btf_dedup_struct_type(d, i);
+ for (i = 0; i < d->btf->nr_types; i++) {
+ err = btf_dedup_struct_type(d, d->btf->start_id + i);
if (err)
return err;
}
@@ -4225,8 +4371,8 @@ static int btf_dedup_ref_types(struct btf_dedup *d)
{
int i, err;
- for (i = 1; i <= d->btf->nr_types; i++) {
- err = btf_dedup_ref_type(d, i);
+ for (i = 0; i < d->btf->nr_types; i++) {
+ err = btf_dedup_ref_type(d, d->btf->start_id + i);
if (err < 0)
return err;
}
@@ -4250,39 +4396,44 @@ static int btf_dedup_ref_types(struct btf_dedup *d)
static int btf_dedup_compact_types(struct btf_dedup *d)
{
__u32 *new_offs;
- __u32 next_type_id = 1;
+ __u32 next_type_id = d->btf->start_id;
+ const struct btf_type *t;
void *p;
- int i, len;
+ int i, id, len;
/* we are going to reuse hypot_map to store compaction remapping */
d->hypot_map[0] = 0;
- for (i = 1; i <= d->btf->nr_types; i++)
- d->hypot_map[i] = BTF_UNPROCESSED_ID;
+ /* base BTF types are not renumbered */
+ for (id = 1; id < d->btf->start_id; id++)
+ d->hypot_map[id] = id;
+ for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++)
+ d->hypot_map[id] = BTF_UNPROCESSED_ID;
p = d->btf->types_data;
- for (i = 1; i <= d->btf->nr_types; i++) {
- if (d->map[i] != i)
+ for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++) {
+ if (d->map[id] != id)
continue;
- len = btf_type_size(btf__type_by_id(d->btf, i));
+ t = btf__type_by_id(d->btf, id);
+ len = btf_type_size(t);
if (len < 0)
return len;
- memmove(p, btf__type_by_id(d->btf, i), len);
- d->hypot_map[i] = next_type_id;
- d->btf->type_offs[next_type_id] = p - d->btf->types_data;
+ memmove(p, t, len);
+ d->hypot_map[id] = next_type_id;
+ d->btf->type_offs[next_type_id - d->btf->start_id] = p - d->btf->types_data;
p += len;
next_type_id++;
}
/* shrink struct btf's internal types index and update btf_header */
- d->btf->nr_types = next_type_id - 1;
- d->btf->type_offs_cap = d->btf->nr_types + 1;
+ d->btf->nr_types = next_type_id - d->btf->start_id;
+ d->btf->type_offs_cap = d->btf->nr_types;
d->btf->hdr->type_len = p - d->btf->types_data;
new_offs = libbpf_reallocarray(d->btf->type_offs, d->btf->type_offs_cap,
sizeof(*new_offs));
- if (!new_offs)
+ if (d->btf->type_offs_cap && !new_offs)
return -ENOMEM;
d->btf->type_offs = new_offs;
d->btf->hdr->str_off = d->btf->hdr->type_len;
@@ -4414,8 +4565,8 @@ static int btf_dedup_remap_types(struct btf_dedup *d)
{
int i, r;
- for (i = 1; i <= d->btf->nr_types; i++) {
- r = btf_dedup_remap_type(d, i);
+ for (i = 0; i < d->btf->nr_types; i++) {
+ r = btf_dedup_remap_type(d, d->btf->start_id + i);
if (r < 0)
return r;
}
diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h
index 57247240a20a..1237bcd1dd17 100644
--- a/tools/lib/bpf/btf.h
+++ b/tools/lib/bpf/btf.h
@@ -31,11 +31,19 @@ enum btf_endianness {
};
LIBBPF_API void btf__free(struct btf *btf);
+
LIBBPF_API struct btf *btf__new(const void *data, __u32 size);
+LIBBPF_API struct btf *btf__new_split(const void *data, __u32 size, struct btf *base_btf);
LIBBPF_API struct btf *btf__new_empty(void);
+LIBBPF_API struct btf *btf__new_empty_split(struct btf *base_btf);
+
LIBBPF_API struct btf *btf__parse(const char *path, struct btf_ext **btf_ext);
+LIBBPF_API struct btf *btf__parse_split(const char *path, struct btf *base_btf);
LIBBPF_API struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext);
+LIBBPF_API struct btf *btf__parse_elf_split(const char *path, struct btf *base_btf);
LIBBPF_API struct btf *btf__parse_raw(const char *path);
+LIBBPF_API struct btf *btf__parse_raw_split(const char *path, struct btf *base_btf);
+
LIBBPF_API int btf__finalize_data(struct bpf_object *obj, struct btf *btf);
LIBBPF_API int btf__load(struct btf *btf);
LIBBPF_API __s32 btf__find_by_name(const struct btf *btf,
@@ -43,6 +51,7 @@ LIBBPF_API __s32 btf__find_by_name(const struct btf *btf,
LIBBPF_API __s32 btf__find_by_name_kind(const struct btf *btf,
const char *type_name, __u32 kind);
LIBBPF_API __u32 btf__get_nr_types(const struct btf *btf);
+LIBBPF_API const struct btf *btf__base_btf(const struct btf *btf);
LIBBPF_API const struct btf_type *btf__type_by_id(const struct btf *btf,
__u32 id);
LIBBPF_API size_t btf__pointer_size(const struct btf *btf);
diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
index 28baee7ba1ca..6ae748f6ea11 100644
--- a/tools/lib/bpf/libbpf.c
+++ b/tools/lib/bpf/libbpf.c
@@ -176,6 +176,8 @@ enum kern_feature_id {
FEAT_PROBE_READ_KERN,
/* BPF_PROG_BIND_MAP is supported */
FEAT_PROG_BIND_MAP,
+ /* Kernel support for module BTFs */
+ FEAT_MODULE_BTF,
__FEAT_CNT,
};
@@ -276,6 +278,7 @@ struct bpf_program {
enum bpf_prog_type type;
enum bpf_attach_type expected_attach_type;
int prog_ifindex;
+ __u32 attach_btf_obj_fd;
__u32 attach_btf_id;
__u32 attach_prog_fd;
void *func_info;
@@ -402,6 +405,13 @@ struct extern_desc {
static LIST_HEAD(bpf_objects_list);
+struct module_btf {
+ struct btf *btf;
+ char *name;
+ __u32 id;
+ int fd;
+};
+
struct bpf_object {
char name[BPF_OBJ_NAME_LEN];
char license[64];
@@ -462,11 +472,19 @@ struct bpf_object {
struct list_head list;
struct btf *btf;
+ struct btf_ext *btf_ext;
+
/* Parse and load BTF vmlinux if any of the programs in the object need
* it at load time.
*/
struct btf *btf_vmlinux;
- struct btf_ext *btf_ext;
+ /* vmlinux BTF override for CO-RE relocations */
+ struct btf *btf_vmlinux_override;
+ /* Lazily initialized kernel module BTFs */
+ struct module_btf *btf_modules;
+ bool btf_modules_loaded;
+ size_t btf_module_cnt;
+ size_t btf_module_cap;
void *priv;
bpf_object_clear_priv_t clear_priv;
@@ -2500,7 +2518,7 @@ static int bpf_object__finalize_btf(struct bpf_object *obj)
return 0;
}
-static inline bool libbpf_prog_needs_vmlinux_btf(struct bpf_program *prog)
+static bool prog_needs_vmlinux_btf(struct bpf_program *prog)
{
if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
prog->type == BPF_PROG_TYPE_LSM)
@@ -2515,37 +2533,43 @@ static inline bool libbpf_prog_needs_vmlinux_btf(struct bpf_program *prog)
return false;
}
-static int bpf_object__load_vmlinux_btf(struct bpf_object *obj)
+static bool obj_needs_vmlinux_btf(const struct bpf_object *obj)
{
- bool need_vmlinux_btf = false;
struct bpf_program *prog;
- int i, err;
+ int i;
/* CO-RE relocations need kernel BTF */
if (obj->btf_ext && obj->btf_ext->core_relo_info.len)
- need_vmlinux_btf = true;
+ return true;
/* Support for typed ksyms needs kernel BTF */
for (i = 0; i < obj->nr_extern; i++) {
const struct extern_desc *ext;
ext = &obj->externs[i];
- if (ext->type == EXT_KSYM && ext->ksym.type_id) {
- need_vmlinux_btf = true;
- break;
- }
+ if (ext->type == EXT_KSYM && ext->ksym.type_id)
+ return true;
}
bpf_object__for_each_program(prog, obj) {
if (!prog->load)
continue;
- if (libbpf_prog_needs_vmlinux_btf(prog)) {
- need_vmlinux_btf = true;
- break;
- }
+ if (prog_needs_vmlinux_btf(prog))
+ return true;
}
- if (!need_vmlinux_btf)
+ return false;
+}
+
+static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force)
+{
+ int err;
+
+ /* btf_vmlinux could be loaded earlier */
+ if (obj->btf_vmlinux)
+ return 0;
+
+ if (!force && !obj_needs_vmlinux_btf(obj))
return 0;
obj->btf_vmlinux = libbpf_find_kernel_btf();
@@ -3960,6 +3984,35 @@ static int probe_prog_bind_map(void)
return ret >= 0;
}
+static int probe_module_btf(void)
+{
+ static const char strs[] = "\0int";
+ __u32 types[] = {
+ /* int */
+ BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
+ };
+ struct bpf_btf_info info;
+ __u32 len = sizeof(info);
+ char name[16];
+ int fd, err;
+
+ fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs));
+ if (fd < 0)
+ return 0; /* BTF not supported at all */
+
+ memset(&info, 0, sizeof(info));
+ info.name = ptr_to_u64(name);
+ info.name_len = sizeof(name);
+
+ /* check that BPF_OBJ_GET_INFO_BY_FD supports specifying name pointer;
+ * kernel's module BTF support coincides with support for
+ * name/name_len fields in struct bpf_btf_info.
+ */
+ err = bpf_obj_get_info_by_fd(fd, &info, &len);
+ close(fd);
+ return !err;
+}
+
enum kern_feature_result {
FEAT_UNKNOWN = 0,
FEAT_SUPPORTED = 1,
@@ -4003,7 +4056,10 @@ static struct kern_feature_desc {
},
[FEAT_PROG_BIND_MAP] = {
"BPF_PROG_BIND_MAP support", probe_prog_bind_map,
- }
+ },
+ [FEAT_MODULE_BTF] = {
+ "module BTF support", probe_module_btf,
+ },
};
static bool kernel_supports(enum kern_feature_id feat_id)
@@ -4603,46 +4659,43 @@ static size_t bpf_core_essential_name_len(const char *name)
return n;
}
-/* dynamically sized list of type IDs */
-struct ids_vec {
- __u32 *data;
+struct core_cand
+{
+ const struct btf *btf;
+ const struct btf_type *t;
+ const char *name;
+ __u32 id;
+};
+
+/* dynamically sized list of type IDs and its associated struct btf */
+struct core_cand_list {
+ struct core_cand *cands;
int len;
};
-static void bpf_core_free_cands(struct ids_vec *cand_ids)
+static void bpf_core_free_cands(struct core_cand_list *cands)
{
- free(cand_ids->data);
- free(cand_ids);
+ free(cands->cands);
+ free(cands);
}
-static struct ids_vec *bpf_core_find_cands(const struct btf *local_btf,
- __u32 local_type_id,
- const struct btf *targ_btf)
+static int bpf_core_add_cands(struct core_cand *local_cand,
+ size_t local_essent_len,
+ const struct btf *targ_btf,
+ const char *targ_btf_name,
+ int targ_start_id,
+ struct core_cand_list *cands)
{
- size_t local_essent_len, targ_essent_len;
- const char *local_name, *targ_name;
- const struct btf_type *t, *local_t;
- struct ids_vec *cand_ids;
- __u32 *new_ids;
- int i, err, n;
-
- local_t = btf__type_by_id(local_btf, local_type_id);
- if (!local_t)
- return ERR_PTR(-EINVAL);
-
- local_name = btf__name_by_offset(local_btf, local_t->name_off);
- if (str_is_empty(local_name))
- return ERR_PTR(-EINVAL);
- local_essent_len = bpf_core_essential_name_len(local_name);
-
- cand_ids = calloc(1, sizeof(*cand_ids));
- if (!cand_ids)
- return ERR_PTR(-ENOMEM);
+ struct core_cand *new_cands, *cand;
+ const struct btf_type *t;
+ const char *targ_name;
+ size_t targ_essent_len;
+ int n, i;
n = btf__get_nr_types(targ_btf);
- for (i = 1; i <= n; i++) {
+ for (i = targ_start_id; i <= n; i++) {
t = btf__type_by_id(targ_btf, i);
- if (btf_kind(t) != btf_kind(local_t))
+ if (btf_kind(t) != btf_kind(local_cand->t))
continue;
targ_name = btf__name_by_offset(targ_btf, t->name_off);
@@ -4653,24 +4706,174 @@ static struct ids_vec *bpf_core_find_cands(const struct btf *local_btf,
if (targ_essent_len != local_essent_len)
continue;
- if (strncmp(local_name, targ_name, local_essent_len) == 0) {
- pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s\n",
- local_type_id, btf_kind_str(local_t),
- local_name, i, btf_kind_str(t), targ_name);
- new_ids = libbpf_reallocarray(cand_ids->data,
- cand_ids->len + 1,
- sizeof(*cand_ids->data));
- if (!new_ids) {
- err = -ENOMEM;
- goto err_out;
- }
- cand_ids->data = new_ids;
- cand_ids->data[cand_ids->len++] = i;
+ if (strncmp(local_cand->name, targ_name, local_essent_len) != 0)
+ continue;
+
+ pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n",
+ local_cand->id, btf_kind_str(local_cand->t),
+ local_cand->name, i, btf_kind_str(t), targ_name,
+ targ_btf_name);
+ new_cands = libbpf_reallocarray(cands->cands, cands->len + 1,
+ sizeof(*cands->cands));
+ if (!new_cands)
+ return -ENOMEM;
+
+ cand = &new_cands[cands->len];
+ cand->btf = targ_btf;
+ cand->t = t;
+ cand->name = targ_name;
+ cand->id = i;
+
+ cands->cands = new_cands;
+ cands->len++;
+ }
+ return 0;
+}
+
+static int load_module_btfs(struct bpf_object *obj)
+{
+ struct bpf_btf_info info;
+ struct module_btf *mod_btf;
+ struct btf *btf;
+ char name[64];
+ __u32 id = 0, len;
+ int err, fd;
+
+ if (obj->btf_modules_loaded)
+ return 0;
+
+ /* don't do this again, even if we find no module BTFs */
+ obj->btf_modules_loaded = true;
+
+ /* kernel too old to support module BTFs */
+ if (!kernel_supports(FEAT_MODULE_BTF))
+ return 0;
+
+ while (true) {
+ err = bpf_btf_get_next_id(id, &id);
+ if (err && errno == ENOENT)
+ return 0;
+ if (err) {
+ err = -errno;
+ pr_warn("failed to iterate BTF objects: %d\n", err);
+ return err;
+ }
+
+ fd = bpf_btf_get_fd_by_id(id);
+ if (fd < 0) {
+ if (errno == ENOENT)
+ continue; /* expected race: BTF was unloaded */
+ err = -errno;
+ pr_warn("failed to get BTF object #%d FD: %d\n", id, err);
+ return err;
}
+
+ len = sizeof(info);
+ memset(&info, 0, sizeof(info));
+ info.name = ptr_to_u64(name);
+ info.name_len = sizeof(name);
+
+ err = bpf_obj_get_info_by_fd(fd, &info, &len);
+ if (err) {
+ err = -errno;
+ pr_warn("failed to get BTF object #%d info: %d\n", id, err);
+ goto err_out;
+ }
+
+ /* ignore non-module BTFs */
+ if (!info.kernel_btf || strcmp(name, "vmlinux") == 0) {
+ close(fd);
+ continue;
+ }
+
+ btf = btf_get_from_fd(fd, obj->btf_vmlinux);
+ if (IS_ERR(btf)) {
+ pr_warn("failed to load module [%s]'s BTF object #%d: %ld\n",
+ name, id, PTR_ERR(btf));
+ err = PTR_ERR(btf);
+ goto err_out;
+ }
+
+ err = btf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,
+ sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);
+ if (err)
+ goto err_out;
+
+ mod_btf = &obj->btf_modules[obj->btf_module_cnt++];
+
+ mod_btf->btf = btf;
+ mod_btf->id = id;
+ mod_btf->fd = fd;
+ mod_btf->name = strdup(name);
+ if (!mod_btf->name) {
+ err = -ENOMEM;
+ goto err_out;
+ }
+ continue;
+
+err_out:
+ close(fd);
+ return err;
+ }
+
+ return 0;
+}
+
+static struct core_cand_list *
+bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id)
+{
+ struct core_cand local_cand = {};
+ struct core_cand_list *cands;
+ const struct btf *main_btf;
+ size_t local_essent_len;
+ int err, i;
+
+ local_cand.btf = local_btf;
+ local_cand.t = btf__type_by_id(local_btf, local_type_id);
+ if (!local_cand.t)
+ return ERR_PTR(-EINVAL);
+
+ local_cand.name = btf__name_by_offset(local_btf, local_cand.t->name_off);
+ if (str_is_empty(local_cand.name))
+ return ERR_PTR(-EINVAL);
+ local_essent_len = bpf_core_essential_name_len(local_cand.name);
+
+ cands = calloc(1, sizeof(*cands));
+ if (!cands)
+ return ERR_PTR(-ENOMEM);
+
+ /* Attempt to find target candidates in vmlinux BTF first */
+ main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux;
+ err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands);
+ if (err)
+ goto err_out;
+
+ /* if vmlinux BTF has any candidate, don't got for module BTFs */
+ if (cands->len)
+ return cands;
+
+ /* if vmlinux BTF was overridden, don't attempt to load module BTFs */
+ if (obj->btf_vmlinux_override)
+ return cands;
+
+ /* now look through module BTFs, trying to still find candidates */
+ err = load_module_btfs(obj);
+ if (err)
+ goto err_out;
+
+ for (i = 0; i < obj->btf_module_cnt; i++) {
+ err = bpf_core_add_cands(&local_cand, local_essent_len,
+ obj->btf_modules[i].btf,
+ obj->btf_modules[i].name,
+ btf__get_nr_types(obj->btf_vmlinux) + 1,
+ cands);
+ if (err)
+ goto err_out;
}
- return cand_ids;
+
+ return cands;
err_out:
- bpf_core_free_cands(cand_ids);
+ bpf_core_free_cands(cands);
return ERR_PTR(err);
}
@@ -5664,7 +5867,6 @@ static int bpf_core_apply_relo(struct bpf_program *prog,
const struct bpf_core_relo *relo,
int relo_idx,
const struct btf *local_btf,
- const struct btf *targ_btf,
struct hashmap *cand_cache)
{
struct bpf_core_spec local_spec, cand_spec, targ_spec = {};
@@ -5672,8 +5874,8 @@ static int bpf_core_apply_relo(struct bpf_program *prog,
struct bpf_core_relo_res cand_res, targ_res;
const struct btf_type *local_type;
const char *local_name;
- struct ids_vec *cand_ids;
- __u32 local_id, cand_id;
+ struct core_cand_list *cands = NULL;
+ __u32 local_id;
const char *spec_str;
int i, j, err;
@@ -5720,24 +5922,24 @@ static int bpf_core_apply_relo(struct bpf_program *prog,
return -EOPNOTSUPP;
}
- if (!hashmap__find(cand_cache, type_key, (void **)&cand_ids)) {
- cand_ids = bpf_core_find_cands(local_btf, local_id, targ_btf);
- if (IS_ERR(cand_ids)) {
- pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld",
+ if (!hashmap__find(cand_cache, type_key, (void **)&cands)) {
+ cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
+ if (IS_ERR(cands)) {
+ pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n",
prog->name, relo_idx, local_id, btf_kind_str(local_type),
- local_name, PTR_ERR(cand_ids));
- return PTR_ERR(cand_ids);
+ local_name, PTR_ERR(cands));
+ return PTR_ERR(cands);
}
- err = hashmap__set(cand_cache, type_key, cand_ids, NULL, NULL);
+ err = hashmap__set(cand_cache, type_key, cands, NULL, NULL);
if (err) {
- bpf_core_free_cands(cand_ids);
+ bpf_core_free_cands(cands);
return err;
}
}
- for (i = 0, j = 0; i < cand_ids->len; i++) {
- cand_id = cand_ids->data[i];
- err = bpf_core_spec_match(&local_spec, targ_btf, cand_id, &cand_spec);
+ for (i = 0, j = 0; i < cands->len; i++) {
+ err = bpf_core_spec_match(&local_spec, cands->cands[i].btf,
+ cands->cands[i].id, &cand_spec);
if (err < 0) {
pr_warn("prog '%s': relo #%d: error matching candidate #%d ",
prog->name, relo_idx, i);
@@ -5781,7 +5983,7 @@ static int bpf_core_apply_relo(struct bpf_program *prog,
return -EINVAL;
}
- cand_ids->data[j++] = cand_spec.root_type_id;
+ cands->cands[j++] = cands->cands[i];
}
/*
@@ -5793,7 +5995,7 @@ static int bpf_core_apply_relo(struct bpf_program *prog,
* depending on relo's kind.
*/
if (j > 0)
- cand_ids->len = j;
+ cands->len = j;
/*
* If no candidates were found, it might be both a programmer error,
@@ -5837,20 +6039,19 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)
struct hashmap_entry *entry;
struct hashmap *cand_cache = NULL;
struct bpf_program *prog;
- struct btf *targ_btf;
const char *sec_name;
int i, err = 0, insn_idx, sec_idx;
if (obj->btf_ext->core_relo_info.len == 0)
return 0;
- if (targ_btf_path)
- targ_btf = btf__parse(targ_btf_path, NULL);
- else
- targ_btf = obj->btf_vmlinux;
- if (IS_ERR_OR_NULL(targ_btf)) {
- pr_warn("failed to get target BTF: %ld\n", PTR_ERR(targ_btf));
- return PTR_ERR(targ_btf);
+ if (targ_btf_path) {
+ obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);
+ if (IS_ERR_OR_NULL(obj->btf_vmlinux_override)) {
+ err = PTR_ERR(obj->btf_vmlinux_override);
+ pr_warn("failed to parse target BTF: %d\n", err);
+ return err;
+ }
}
cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL);
@@ -5902,8 +6103,7 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)
if (!prog->load)
continue;
- err = bpf_core_apply_relo(prog, rec, i, obj->btf,
- targ_btf, cand_cache);
+ err = bpf_core_apply_relo(prog, rec, i, obj->btf, cand_cache);
if (err) {
pr_warn("prog '%s': relo #%d: failed to relocate: %d\n",
prog->name, i, err);
@@ -5913,9 +6113,10 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)
}
out:
- /* obj->btf_vmlinux is freed at the end of object load phase */
- if (targ_btf != obj->btf_vmlinux)
- btf__free(targ_btf);
+ /* obj->btf_vmlinux and module BTFs are freed after object load */
+ btf__free(obj->btf_vmlinux_override);
+ obj->btf_vmlinux_override = NULL;
+
if (!IS_ERR_OR_NULL(cand_cache)) {
hashmap__for_each_entry(cand_cache, entry, i) {
bpf_core_free_cands(entry->value);
@@ -6626,16 +6827,25 @@ static int
load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt,
char *license, __u32 kern_version, int *pfd)
{
- struct bpf_load_program_attr load_attr;
+ struct bpf_prog_load_params load_attr = {};
char *cp, errmsg[STRERR_BUFSIZE];
size_t log_buf_size = 0;
char *log_buf = NULL;
int btf_fd, ret;
+ if (prog->type == BPF_PROG_TYPE_UNSPEC) {
+ /*
+ * The program type must be set. Most likely we couldn't find a proper
+ * section definition at load time, and thus we didn't infer the type.
+ */
+ pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n",
+ prog->name, prog->sec_name);
+ return -EINVAL;
+ }
+
if (!insns || !insns_cnt)
return -EINVAL;
- memset(&load_attr, 0, sizeof(struct bpf_load_program_attr));
load_attr.prog_type = prog->type;
/* old kernels might not support specifying expected_attach_type */
if (!kernel_supports(FEAT_EXP_ATTACH_TYPE) && prog->sec_def &&
@@ -6646,19 +6856,17 @@ load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt,
if (kernel_supports(FEAT_PROG_NAME))
load_attr.name = prog->name;
load_attr.insns = insns;
- load_attr.insns_cnt = insns_cnt;
+ load_attr.insn_cnt = insns_cnt;
load_attr.license = license;
- if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
- prog->type == BPF_PROG_TYPE_LSM) {
- load_attr.attach_btf_id = prog->attach_btf_id;
- } else if (prog->type == BPF_PROG_TYPE_TRACING ||
- prog->type == BPF_PROG_TYPE_EXT) {
+ load_attr.attach_btf_id = prog->attach_btf_id;
+ if (prog->attach_prog_fd)
load_attr.attach_prog_fd = prog->attach_prog_fd;
- load_attr.attach_btf_id = prog->attach_btf_id;
- } else {
- load_attr.kern_version = kern_version;
- load_attr.prog_ifindex = prog->prog_ifindex;
- }
+ else
+ load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
+ load_attr.attach_btf_id = prog->attach_btf_id;
+ load_attr.kern_version = kern_version;
+ load_attr.prog_ifindex = prog->prog_ifindex;
+
/* specify func_info/line_info only if kernel supports them */
btf_fd = bpf_object__btf_fd(prog->obj);
if (btf_fd >= 0 && kernel_supports(FEAT_BTF_FUNC)) {
@@ -6682,7 +6890,9 @@ retry_load:
*log_buf = 0;
}
- ret = bpf_load_program_xattr(&load_attr, log_buf, log_buf_size);
+ load_attr.log_buf = log_buf;
+ load_attr.log_buf_sz = log_buf_size;
+ ret = libbpf__bpf_prog_load(&load_attr);
if (ret >= 0) {
if (log_buf && load_attr.log_level)
@@ -6723,9 +6933,9 @@ retry_load:
pr_warn("-- BEGIN DUMP LOG ---\n");
pr_warn("\n%s\n", log_buf);
pr_warn("-- END LOG --\n");
- } else if (load_attr.insns_cnt >= BPF_MAXINSNS) {
+ } else if (load_attr.insn_cnt >= BPF_MAXINSNS) {
pr_warn("Program too large (%zu insns), at most %d insns\n",
- load_attr.insns_cnt, BPF_MAXINSNS);
+ load_attr.insn_cnt, BPF_MAXINSNS);
ret = -LIBBPF_ERRNO__PROG2BIG;
} else if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) {
/* Wrong program type? */
@@ -6733,7 +6943,9 @@ retry_load:
load_attr.prog_type = BPF_PROG_TYPE_KPROBE;
load_attr.expected_attach_type = 0;
- fd = bpf_load_program_xattr(&load_attr, NULL, 0);
+ load_attr.log_buf = NULL;
+ load_attr.log_buf_sz = 0;
+ fd = libbpf__bpf_prog_load(&load_attr);
if (fd >= 0) {
close(fd);
ret = -LIBBPF_ERRNO__PROGTYPE;
@@ -6746,11 +6958,11 @@ out:
return ret;
}
-static int libbpf_find_attach_btf_id(struct bpf_program *prog);
+static int libbpf_find_attach_btf_id(struct bpf_program *prog, int *btf_obj_fd, int *btf_type_id);
int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_ver)
{
- int err = 0, fd, i, btf_id;
+ int err = 0, fd, i;
if (prog->obj->loaded) {
pr_warn("prog '%s': can't load after object was loaded\n", prog->name);
@@ -6760,10 +6972,14 @@ int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_ver)
if ((prog->type == BPF_PROG_TYPE_TRACING ||
prog->type == BPF_PROG_TYPE_LSM ||
prog->type == BPF_PROG_TYPE_EXT) && !prog->attach_btf_id) {
- btf_id = libbpf_find_attach_btf_id(prog);
- if (btf_id <= 0)
- return btf_id;
- prog->attach_btf_id = btf_id;
+ int btf_obj_fd = 0, btf_type_id = 0;
+
+ err = libbpf_find_attach_btf_id(prog, &btf_obj_fd, &btf_type_id);
+ if (err)
+ return err;
+
+ prog->attach_btf_obj_fd = btf_obj_fd;
+ prog->attach_btf_id = btf_type_id;
}
if (prog->instances.nr < 0 || !prog->instances.fds) {
@@ -6923,9 +7139,12 @@ __bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz,
bpf_object__for_each_program(prog, obj) {
prog->sec_def = find_sec_def(prog->sec_name);
- if (!prog->sec_def)
+ if (!prog->sec_def) {
/* couldn't guess, but user might manually specify */
+ pr_debug("prog '%s': unrecognized ELF section name '%s'\n",
+ prog->name, prog->sec_name);
continue;
+ }
if (prog->sec_def->is_sleepable)
prog->prog_flags |= BPF_F_SLEEPABLE;
@@ -7262,7 +7481,7 @@ int bpf_object__load_xattr(struct bpf_object_load_attr *attr)
}
err = bpf_object__probe_loading(obj);
- err = err ? : bpf_object__load_vmlinux_btf(obj);
+ err = err ? : bpf_object__load_vmlinux_btf(obj, false);
err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
err = err ? : bpf_object__sanitize_and_load_btf(obj);
err = err ? : bpf_object__sanitize_maps(obj);
@@ -7271,6 +7490,15 @@ int bpf_object__load_xattr(struct bpf_object_load_attr *attr)
err = err ? : bpf_object__relocate(obj, attr->target_btf_path);
err = err ? : bpf_object__load_progs(obj, attr->log_level);
+ /* clean up module BTFs */
+ for (i = 0; i < obj->btf_module_cnt; i++) {
+ close(obj->btf_modules[i].fd);
+ btf__free(obj->btf_modules[i].btf);
+ free(obj->btf_modules[i].name);
+ }
+ free(obj->btf_modules);
+
+ /* clean up vmlinux BTF */
btf__free(obj->btf_vmlinux);
obj->btf_vmlinux = NULL;
@@ -7649,6 +7877,16 @@ bool bpf_map__is_pinned(const struct bpf_map *map)
return map->pinned;
}
+static void sanitize_pin_path(char *s)
+{
+ /* bpffs disallows periods in path names */
+ while (*s) {
+ if (*s == '.')
+ *s = '_';
+ s++;
+ }
+}
+
int bpf_object__pin_maps(struct bpf_object *obj, const char *path)
{
struct bpf_map *map;
@@ -7678,6 +7916,7 @@ int bpf_object__pin_maps(struct bpf_object *obj, const char *path)
err = -ENAMETOOLONG;
goto err_unpin_maps;
}
+ sanitize_pin_path(buf);
pin_path = buf;
} else if (!map->pin_path) {
continue;
@@ -7722,6 +7961,7 @@ int bpf_object__unpin_maps(struct bpf_object *obj, const char *path)
return -EINVAL;
else if (len >= PATH_MAX)
return -ENAMETOOLONG;
+ sanitize_pin_path(buf);
pin_path = buf;
} else if (!map->pin_path) {
continue;
@@ -8607,8 +8847,8 @@ static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
return btf__find_by_name_kind(btf, btf_type_name, kind);
}
-static inline int __find_vmlinux_btf_id(struct btf *btf, const char *name,
- enum bpf_attach_type attach_type)
+static inline int find_attach_btf_id(struct btf *btf, const char *name,
+ enum bpf_attach_type attach_type)
{
int err;
@@ -8624,9 +8864,6 @@ static inline int __find_vmlinux_btf_id(struct btf *btf, const char *name,
else
err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
- if (err <= 0)
- pr_warn("%s is not found in vmlinux BTF\n", name);
-
return err;
}
@@ -8642,7 +8879,10 @@ int libbpf_find_vmlinux_btf_id(const char *name,
return -EINVAL;
}
- err = __find_vmlinux_btf_id(btf, name, attach_type);
+ err = find_attach_btf_id(btf, name, attach_type);
+ if (err <= 0)
+ pr_warn("%s is not found in vmlinux BTF\n", name);
+
btf__free(btf);
return err;
}
@@ -8680,11 +8920,49 @@ out:
return err;
}
-static int libbpf_find_attach_btf_id(struct bpf_program *prog)
+static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name,
+ enum bpf_attach_type attach_type,
+ int *btf_obj_fd, int *btf_type_id)
+{
+ int ret, i;
+
+ ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type);
+ if (ret > 0) {
+ *btf_obj_fd = 0; /* vmlinux BTF */
+ *btf_type_id = ret;
+ return 0;
+ }
+ if (ret != -ENOENT)
+ return ret;
+
+ ret = load_module_btfs(obj);
+ if (ret)
+ return ret;
+
+ for (i = 0; i < obj->btf_module_cnt; i++) {
+ const struct module_btf *mod = &obj->btf_modules[i];
+
+ ret = find_attach_btf_id(mod->btf, attach_name, attach_type);
+ if (ret > 0) {
+ *btf_obj_fd = mod->fd;
+ *btf_type_id = ret;
+ return 0;
+ }
+ if (ret == -ENOENT)
+ continue;
+
+ return ret;
+ }
+
+ return -ESRCH;
+}
+
+static int libbpf_find_attach_btf_id(struct bpf_program *prog, int *btf_obj_fd, int *btf_type_id)
{
enum bpf_attach_type attach_type = prog->expected_attach_type;
__u32 attach_prog_fd = prog->attach_prog_fd;
- const char *name = prog->sec_name;
+ const char *name = prog->sec_name, *attach_name;
+ const struct bpf_sec_def *sec = NULL;
int i, err;
if (!name)
@@ -8695,17 +8973,37 @@ static int libbpf_find_attach_btf_id(struct bpf_program *prog)
continue;
if (strncmp(name, section_defs[i].sec, section_defs[i].len))
continue;
- if (attach_prog_fd)
- err = libbpf_find_prog_btf_id(name + section_defs[i].len,
- attach_prog_fd);
- else
- err = __find_vmlinux_btf_id(prog->obj->btf_vmlinux,
- name + section_defs[i].len,
- attach_type);
+
+ sec = &section_defs[i];
+ break;
+ }
+
+ if (!sec) {
+ pr_warn("failed to identify BTF ID based on ELF section name '%s'\n", name);
+ return -ESRCH;
+ }
+ attach_name = name + sec->len;
+
+ /* BPF program's BTF ID */
+ if (attach_prog_fd) {
+ err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd);
+ if (err < 0) {
+ pr_warn("failed to find BPF program (FD %d) BTF ID for '%s': %d\n",
+ attach_prog_fd, attach_name, err);
+ return err;
+ }
+ *btf_obj_fd = 0;
+ *btf_type_id = err;
+ return 0;
+ }
+
+ /* kernel/module BTF ID */
+ err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id);
+ if (err) {
+ pr_warn("failed to find kernel BTF type ID of '%s': %d\n", attach_name, err);
return err;
}
- pr_warn("failed to identify btf_id based on ELF section name '%s'\n", name);
- return -ESRCH;
+ return 0;
}
int libbpf_attach_type_by_name(const char *name,
@@ -10578,22 +10876,33 @@ int bpf_program__set_attach_target(struct bpf_program *prog,
int attach_prog_fd,
const char *attach_func_name)
{
- int btf_id;
+ int btf_obj_fd = 0, btf_id = 0, err;
if (!prog || attach_prog_fd < 0 || !attach_func_name)
return -EINVAL;
- if (attach_prog_fd)
+ if (prog->obj->loaded)
+ return -EINVAL;
+
+ if (attach_prog_fd) {
btf_id = libbpf_find_prog_btf_id(attach_func_name,
attach_prog_fd);
- else
- btf_id = libbpf_find_vmlinux_btf_id(attach_func_name,
- prog->expected_attach_type);
-
- if (btf_id < 0)
- return btf_id;
+ if (btf_id < 0)
+ return btf_id;
+ } else {
+ /* load btf_vmlinux, if not yet */
+ err = bpf_object__load_vmlinux_btf(prog->obj, true);
+ if (err)
+ return err;
+ err = find_kernel_btf_id(prog->obj, attach_func_name,
+ prog->expected_attach_type,
+ &btf_obj_fd, &btf_id);
+ if (err)
+ return err;
+ }
prog->attach_btf_id = btf_id;
+ prog->attach_btf_obj_fd = btf_obj_fd;
prog->attach_prog_fd = attach_prog_fd;
return 0;
}
diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h
index 6909ee81113a..3c35eb401931 100644
--- a/tools/lib/bpf/libbpf.h
+++ b/tools/lib/bpf/libbpf.h
@@ -536,6 +536,7 @@ LIBBPF_API int ring_buffer__add(struct ring_buffer *rb, int map_fd,
ring_buffer_sample_fn sample_cb, void *ctx);
LIBBPF_API int ring_buffer__poll(struct ring_buffer *rb, int timeout_ms);
LIBBPF_API int ring_buffer__consume(struct ring_buffer *rb);
+LIBBPF_API int ring_buffer__epoll_fd(const struct ring_buffer *rb);
/* Perf buffer APIs */
struct perf_buffer;
diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map
index 4ebfadf45b47..1c0fd2dd233a 100644
--- a/tools/lib/bpf/libbpf.map
+++ b/tools/lib/bpf/libbpf.map
@@ -337,3 +337,16 @@ LIBBPF_0.2.0 {
perf_buffer__consume_buffer;
xsk_socket__create_shared;
} LIBBPF_0.1.0;
+
+LIBBPF_0.3.0 {
+ global:
+ btf__base_btf;
+ btf__parse_elf_split;
+ btf__parse_raw_split;
+ btf__parse_split;
+ btf__new_empty_split;
+ btf__new_split;
+ ring_buffer__epoll_fd;
+ xsk_setup_xdp_prog;
+ xsk_socket__update_xskmap;
+} LIBBPF_0.2.0;
diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h
index d99bc847bf84..969d0ac592ba 100644
--- a/tools/lib/bpf/libbpf_internal.h
+++ b/tools/lib/bpf/libbpf_internal.h
@@ -151,10 +151,41 @@ int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz);
int libbpf__load_raw_btf(const char *raw_types, size_t types_len,
const char *str_sec, size_t str_len);
+struct bpf_prog_load_params {
+ enum bpf_prog_type prog_type;
+ enum bpf_attach_type expected_attach_type;
+ const char *name;
+ const struct bpf_insn *insns;
+ size_t insn_cnt;
+ const char *license;
+ __u32 kern_version;
+ __u32 attach_prog_fd;
+ __u32 attach_btf_obj_fd;
+ __u32 attach_btf_id;
+ __u32 prog_ifindex;
+ __u32 prog_btf_fd;
+ __u32 prog_flags;
+
+ __u32 func_info_rec_size;
+ const void *func_info;
+ __u32 func_info_cnt;
+
+ __u32 line_info_rec_size;
+ const void *line_info;
+ __u32 line_info_cnt;
+
+ __u32 log_level;
+ char *log_buf;
+ size_t log_buf_sz;
+};
+
+int libbpf__bpf_prog_load(const struct bpf_prog_load_params *load_attr);
+
int bpf_object__section_size(const struct bpf_object *obj, const char *name,
__u32 *size);
int bpf_object__variable_offset(const struct bpf_object *obj, const char *name,
__u32 *off);
+struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf);
struct btf_ext_info {
/*
diff --git a/tools/lib/bpf/libbpf_probes.c b/tools/lib/bpf/libbpf_probes.c
index 5482a9b7ae2d..ecaae2927ab8 100644
--- a/tools/lib/bpf/libbpf_probes.c
+++ b/tools/lib/bpf/libbpf_probes.c
@@ -230,6 +230,7 @@ bool bpf_probe_map_type(enum bpf_map_type map_type, __u32 ifindex)
break;
case BPF_MAP_TYPE_SK_STORAGE:
case BPF_MAP_TYPE_INODE_STORAGE:
+ case BPF_MAP_TYPE_TASK_STORAGE:
btf_key_type_id = 1;
btf_value_type_id = 3;
value_size = 8;
diff --git a/tools/lib/bpf/ringbuf.c b/tools/lib/bpf/ringbuf.c
index 98537ff2679e..8caaafe7e312 100644
--- a/tools/lib/bpf/ringbuf.c
+++ b/tools/lib/bpf/ringbuf.c
@@ -282,3 +282,9 @@ int ring_buffer__poll(struct ring_buffer *rb, int timeout_ms)
}
return cnt < 0 ? -errno : res;
}
+
+/* Get an fd that can be used to sleep until data is available in the ring(s) */
+int ring_buffer__epoll_fd(const struct ring_buffer *rb)
+{
+ return rb->epoll_fd;
+}
diff --git a/tools/lib/bpf/xsk.c b/tools/lib/bpf/xsk.c
index 9bc537d0b92d..e3e41ceeb1bc 100644
--- a/tools/lib/bpf/xsk.c
+++ b/tools/lib/bpf/xsk.c
@@ -566,8 +566,35 @@ static int xsk_set_bpf_maps(struct xsk_socket *xsk)
&xsk->fd, 0);
}
-static int xsk_setup_xdp_prog(struct xsk_socket *xsk)
+static int xsk_create_xsk_struct(int ifindex, struct xsk_socket *xsk)
{
+ char ifname[IFNAMSIZ];
+ struct xsk_ctx *ctx;
+ char *interface;
+
+ ctx = calloc(1, sizeof(*ctx));
+ if (!ctx)
+ return -ENOMEM;
+
+ interface = if_indextoname(ifindex, &ifname[0]);
+ if (!interface) {
+ free(ctx);
+ return -errno;
+ }
+
+ ctx->ifindex = ifindex;
+ memcpy(ctx->ifname, ifname, IFNAMSIZ -1);
+ ctx->ifname[IFNAMSIZ - 1] = 0;
+
+ xsk->ctx = ctx;
+
+ return 0;
+}
+
+static int __xsk_setup_xdp_prog(struct xsk_socket *_xdp,
+ int *xsks_map_fd)
+{
+ struct xsk_socket *xsk = _xdp;
struct xsk_ctx *ctx = xsk->ctx;
__u32 prog_id = 0;
int err;
@@ -584,8 +611,7 @@ static int xsk_setup_xdp_prog(struct xsk_socket *xsk)
err = xsk_load_xdp_prog(xsk);
if (err) {
- xsk_delete_bpf_maps(xsk);
- return err;
+ goto err_load_xdp_prog;
}
} else {
ctx->prog_fd = bpf_prog_get_fd_by_id(prog_id);
@@ -598,15 +624,29 @@ static int xsk_setup_xdp_prog(struct xsk_socket *xsk)
}
}
- if (xsk->rx)
+ if (xsk->rx) {
err = xsk_set_bpf_maps(xsk);
- if (err) {
- xsk_delete_bpf_maps(xsk);
- close(ctx->prog_fd);
- return err;
+ if (err) {
+ if (!prog_id) {
+ goto err_set_bpf_maps;
+ } else {
+ close(ctx->prog_fd);
+ return err;
+ }
+ }
}
+ if (xsks_map_fd)
+ *xsks_map_fd = ctx->xsks_map_fd;
return 0;
+
+err_set_bpf_maps:
+ close(ctx->prog_fd);
+ bpf_set_link_xdp_fd(ctx->ifindex, -1, 0);
+err_load_xdp_prog:
+ xsk_delete_bpf_maps(xsk);
+
+ return err;
}
static struct xsk_ctx *xsk_get_ctx(struct xsk_umem *umem, int ifindex,
@@ -689,6 +729,40 @@ static struct xsk_ctx *xsk_create_ctx(struct xsk_socket *xsk,
return ctx;
}
+static void xsk_destroy_xsk_struct(struct xsk_socket *xsk)
+{
+ free(xsk->ctx);
+ free(xsk);
+}
+
+int xsk_socket__update_xskmap(struct xsk_socket *xsk, int fd)
+{
+ xsk->ctx->xsks_map_fd = fd;
+ return xsk_set_bpf_maps(xsk);
+}
+
+int xsk_setup_xdp_prog(int ifindex, int *xsks_map_fd)
+{
+ struct xsk_socket *xsk;
+ int res;
+
+ xsk = calloc(1, sizeof(*xsk));
+ if (!xsk)
+ return -ENOMEM;
+
+ res = xsk_create_xsk_struct(ifindex, xsk);
+ if (res) {
+ free(xsk);
+ return -EINVAL;
+ }
+
+ res = __xsk_setup_xdp_prog(xsk, xsks_map_fd);
+
+ xsk_destroy_xsk_struct(xsk);
+
+ return res;
+}
+
int xsk_socket__create_shared(struct xsk_socket **xsk_ptr,
const char *ifname,
__u32 queue_id, struct xsk_umem *umem,
@@ -838,7 +912,7 @@ int xsk_socket__create_shared(struct xsk_socket **xsk_ptr,
ctx->prog_fd = -1;
if (!(xsk->config.libbpf_flags & XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD)) {
- err = xsk_setup_xdp_prog(xsk);
+ err = __xsk_setup_xdp_prog(xsk, NULL);
if (err)
goto out_mmap_tx;
}
diff --git a/tools/lib/bpf/xsk.h b/tools/lib/bpf/xsk.h
index 1069c46364ff..e9f121f5d129 100644
--- a/tools/lib/bpf/xsk.h
+++ b/tools/lib/bpf/xsk.h
@@ -113,8 +113,7 @@ static inline __u32 xsk_cons_nb_avail(struct xsk_ring_cons *r, __u32 nb)
return (entries > nb) ? nb : entries;
}
-static inline size_t xsk_ring_prod__reserve(struct xsk_ring_prod *prod,
- size_t nb, __u32 *idx)
+static inline __u32 xsk_ring_prod__reserve(struct xsk_ring_prod *prod, __u32 nb, __u32 *idx)
{
if (xsk_prod_nb_free(prod, nb) < nb)
return 0;
@@ -125,7 +124,7 @@ static inline size_t xsk_ring_prod__reserve(struct xsk_ring_prod *prod,
return nb;
}
-static inline void xsk_ring_prod__submit(struct xsk_ring_prod *prod, size_t nb)
+static inline void xsk_ring_prod__submit(struct xsk_ring_prod *prod, __u32 nb)
{
/* Make sure everything has been written to the ring before indicating
* this to the kernel by writing the producer pointer.
@@ -135,10 +134,9 @@ static inline void xsk_ring_prod__submit(struct xsk_ring_prod *prod, size_t nb)
*prod->producer += nb;
}
-static inline size_t xsk_ring_cons__peek(struct xsk_ring_cons *cons,
- size_t nb, __u32 *idx)
+static inline __u32 xsk_ring_cons__peek(struct xsk_ring_cons *cons, __u32 nb, __u32 *idx)
{
- size_t entries = xsk_cons_nb_avail(cons, nb);
+ __u32 entries = xsk_cons_nb_avail(cons, nb);
if (entries > 0) {
/* Make sure we do not speculatively read the data before
@@ -153,7 +151,12 @@ static inline size_t xsk_ring_cons__peek(struct xsk_ring_cons *cons,
return entries;
}
-static inline void xsk_ring_cons__release(struct xsk_ring_cons *cons, size_t nb)
+static inline void xsk_ring_cons__cancel(struct xsk_ring_cons *cons, __u32 nb)
+{
+ cons->cached_cons -= nb;
+}
+
+static inline void xsk_ring_cons__release(struct xsk_ring_cons *cons, __u32 nb)
{
/* Make sure data has been read before indicating we are done
* with the entries by updating the consumer pointer.
@@ -201,6 +204,11 @@ struct xsk_umem_config {
__u32 flags;
};
+LIBBPF_API int xsk_setup_xdp_prog(int ifindex,
+ int *xsks_map_fd);
+LIBBPF_API int xsk_socket__update_xskmap(struct xsk_socket *xsk,
+ int xsks_map_fd);
+
/* Flags for the libbpf_flags field. */
#define XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD (1 << 0)
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-28 12:15:52 +0000