summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/trace/Kconfig11
-rw-r--r--kernel/trace/ftrace.c51
-rw-r--r--kernel/trace/trace.c144
-rw-r--r--kernel/trace/trace.h6
-rw-r--r--kernel/trace/trace_branch.c19
-rw-r--r--kernel/trace/trace_events.c81
-rw-r--r--kernel/trace/trace_export.c87
-rw-r--r--kernel/trace/trace_functions_graph.c78
-rw-r--r--kernel/trace/trace_kprobe.c108
-rw-r--r--kernel/trace/trace_syscalls.c113
10 files changed, 370 insertions, 328 deletions
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index 60e2ce0181ee..13e13d428cd3 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -328,15 +328,6 @@ config BRANCH_TRACER
Say N if unsure.
-config POWER_TRACER
- bool "Trace power consumption behavior"
- depends on X86
- select GENERIC_TRACER
- help
- This tracer helps developers to analyze and optimize the kernel's
- power management decisions, specifically the C-state and P-state
- behavior.
-
config KSYM_TRACER
bool "Trace read and write access on kernel memory locations"
depends on HAVE_HW_BREAKPOINT
@@ -449,7 +440,7 @@ config BLK_DEV_IO_TRACE
config KPROBE_EVENT
depends on KPROBES
- depends on X86
+ depends on HAVE_REGS_AND_STACK_ACCESS_API
bool "Enable kprobes-based dynamic events"
select TRACING
default y
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 1e6640f80454..d996353473fd 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -2426,6 +2426,7 @@ static const struct file_operations ftrace_notrace_fops = {
static DEFINE_MUTEX(graph_lock);
int ftrace_graph_count;
+int ftrace_graph_filter_enabled;
unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
static void *
@@ -2448,7 +2449,7 @@ static void *g_start(struct seq_file *m, loff_t *pos)
mutex_lock(&graph_lock);
/* Nothing, tell g_show to print all functions are enabled */
- if (!ftrace_graph_count && !*pos)
+ if (!ftrace_graph_filter_enabled && !*pos)
return (void *)1;
return __g_next(m, pos);
@@ -2494,6 +2495,7 @@ ftrace_graph_open(struct inode *inode, struct file *file)
mutex_lock(&graph_lock);
if ((file->f_mode & FMODE_WRITE) &&
(file->f_flags & O_TRUNC)) {
+ ftrace_graph_filter_enabled = 0;
ftrace_graph_count = 0;
memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
}
@@ -2519,7 +2521,7 @@ ftrace_set_func(unsigned long *array, int *idx, char *buffer)
struct dyn_ftrace *rec;
struct ftrace_page *pg;
int search_len;
- int found = 0;
+ int fail = 1;
int type, not;
char *search;
bool exists;
@@ -2530,37 +2532,51 @@ ftrace_set_func(unsigned long *array, int *idx, char *buffer)
/* decode regex */
type = filter_parse_regex(buffer, strlen(buffer), &search, &not);
- if (not)
- return -EINVAL;
+ if (!not && *idx >= FTRACE_GRAPH_MAX_FUNCS)
+ return -EBUSY;
search_len = strlen(search);
mutex_lock(&ftrace_lock);
do_for_each_ftrace_rec(pg, rec) {
- if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
- break;
-
if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE))
continue;
if (ftrace_match_record(rec, search, search_len, type)) {
- /* ensure it is not already in the array */
+ /* if it is in the array */
exists = false;
- for (i = 0; i < *idx; i++)
+ for (i = 0; i < *idx; i++) {
if (array[i] == rec->ip) {
exists = true;
break;
}
- if (!exists)
- array[(*idx)++] = rec->ip;
- found = 1;
+ }
+
+ if (!not) {
+ fail = 0;
+ if (!exists) {
+ array[(*idx)++] = rec->ip;
+ if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
+ goto out;
+ }
+ } else {
+ if (exists) {
+ array[i] = array[--(*idx)];
+ array[*idx] = 0;
+ fail = 0;
+ }
+ }
}
} while_for_each_ftrace_rec();
-
+out:
mutex_unlock(&ftrace_lock);
- return found ? 0 : -EINVAL;
+ if (fail)
+ return -EINVAL;
+
+ ftrace_graph_filter_enabled = 1;
+ return 0;
}
static ssize_t
@@ -2570,16 +2586,11 @@ ftrace_graph_write(struct file *file, const char __user *ubuf,
struct trace_parser parser;
ssize_t read, ret;
- if (!cnt || cnt < 0)
+ if (!cnt)
return 0;
mutex_lock(&graph_lock);
- if (ftrace_graph_count >= FTRACE_GRAPH_MAX_FUNCS) {
- ret = -EBUSY;
- goto out_unlock;
- }
-
if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) {
ret = -ENOMEM;
goto out_unlock;
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index eac6875cb990..032c57ca6502 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -32,6 +32,7 @@
#include <linux/splice.h>
#include <linux/kdebug.h>
#include <linux/string.h>
+#include <linux/rwsem.h>
#include <linux/ctype.h>
#include <linux/init.h>
#include <linux/poll.h>
@@ -102,9 +103,6 @@ static inline void ftrace_enable_cpu(void)
static cpumask_var_t __read_mostly tracing_buffer_mask;
-/* Define which cpu buffers are currently read in trace_pipe */
-static cpumask_var_t tracing_reader_cpumask;
-
#define for_each_tracing_cpu(cpu) \
for_each_cpu(cpu, tracing_buffer_mask)
@@ -243,12 +241,91 @@ static struct tracer *current_trace __read_mostly;
/*
* trace_types_lock is used to protect the trace_types list.
- * This lock is also used to keep user access serialized.
- * Accesses from userspace will grab this lock while userspace
- * activities happen inside the kernel.
*/
static DEFINE_MUTEX(trace_types_lock);
+/*
+ * serialize the access of the ring buffer
+ *
+ * ring buffer serializes readers, but it is low level protection.
+ * The validity of the events (which returns by ring_buffer_peek() ..etc)
+ * are not protected by ring buffer.
+ *
+ * The content of events may become garbage if we allow other process consumes
+ * these events concurrently:
+ * A) the page of the consumed events may become a normal page
+ * (not reader page) in ring buffer, and this page will be rewrited
+ * by events producer.
+ * B) The page of the consumed events may become a page for splice_read,
+ * and this page will be returned to system.
+ *
+ * These primitives allow multi process access to different cpu ring buffer
+ * concurrently.
+ *
+ * These primitives don't distinguish read-only and read-consume access.
+ * Multi read-only access are also serialized.
+ */
+
+#ifdef CONFIG_SMP
+static DECLARE_RWSEM(all_cpu_access_lock);
+static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
+
+static inline void trace_access_lock(int cpu)
+{
+ if (cpu == TRACE_PIPE_ALL_CPU) {
+ /* gain it for accessing the whole ring buffer. */
+ down_write(&all_cpu_access_lock);
+ } else {
+ /* gain it for accessing a cpu ring buffer. */
+
+ /* Firstly block other trace_access_lock(TRACE_PIPE_ALL_CPU). */
+ down_read(&all_cpu_access_lock);
+
+ /* Secondly block other access to this @cpu ring buffer. */
+ mutex_lock(&per_cpu(cpu_access_lock, cpu));
+ }
+}
+
+static inline void trace_access_unlock(int cpu)
+{
+ if (cpu == TRACE_PIPE_ALL_CPU) {
+ up_write(&all_cpu_access_lock);
+ } else {
+ mutex_unlock(&per_cpu(cpu_access_lock, cpu));
+ up_read(&all_cpu_access_lock);
+ }
+}
+
+static inline void trace_access_lock_init(void)
+{
+ int cpu;
+
+ for_each_possible_cpu(cpu)
+ mutex_init(&per_cpu(cpu_access_lock, cpu));
+}
+
+#else
+
+static DEFINE_MUTEX(access_lock);
+
+static inline void trace_access_lock(int cpu)
+{
+ (void)cpu;
+ mutex_lock(&access_lock);
+}
+
+static inline void trace_access_unlock(int cpu)
+{
+ (void)cpu;
+ mutex_unlock(&access_lock);
+}
+
+static inline void trace_access_lock_init(void)
+{
+}
+
+#endif
+
/* trace_wait is a waitqueue for tasks blocked on trace_poll */
static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
@@ -1320,8 +1397,10 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
entry->fmt = fmt;
memcpy(entry->buf, trace_buf, sizeof(u32) * len);
- if (!filter_check_discard(call, entry, buffer, event))
+ if (!filter_check_discard(call, entry, buffer, event)) {
ring_buffer_unlock_commit(buffer, event);
+ ftrace_trace_stack(buffer, flags, 6, pc);
+ }
out_unlock:
arch_spin_unlock(&trace_buf_lock);
@@ -1394,8 +1473,10 @@ int trace_array_vprintk(struct trace_array *tr,
memcpy(&entry->buf, trace_buf, len);
entry->buf[len] = '\0';
- if (!filter_check_discard(call, entry, buffer, event))
+ if (!filter_check_discard(call, entry, buffer, event)) {
ring_buffer_unlock_commit(buffer, event);
+ ftrace_trace_stack(buffer, irq_flags, 6, pc);
+ }
out_unlock:
arch_spin_unlock(&trace_buf_lock);
@@ -1585,12 +1666,6 @@ static void tracing_iter_reset(struct trace_iterator *iter, int cpu)
}
/*
- * No necessary locking here. The worst thing which can
- * happen is loosing events consumed at the same time
- * by a trace_pipe reader.
- * Other than that, we don't risk to crash the ring buffer
- * because it serializes the readers.
- *
* The current tracer is copied to avoid a global locking
* all around.
*/
@@ -1645,12 +1720,16 @@ static void *s_start(struct seq_file *m, loff_t *pos)
}
trace_event_read_lock();
+ trace_access_lock(cpu_file);
return p;
}
static void s_stop(struct seq_file *m, void *p)
{
+ struct trace_iterator *iter = m->private;
+
atomic_dec(&trace_record_cmdline_disabled);
+ trace_access_unlock(iter->cpu_file);
trace_event_read_unlock();
}
@@ -2841,22 +2920,6 @@ static int tracing_open_pipe(struct inode *inode, struct file *filp)
mutex_lock(&trace_types_lock);
- /* We only allow one reader per cpu */
- if (cpu_file == TRACE_PIPE_ALL_CPU) {
- if (!cpumask_empty(tracing_reader_cpumask)) {
- ret = -EBUSY;
- goto out;
- }
- cpumask_setall(tracing_reader_cpumask);
- } else {
- if (!cpumask_test_cpu(cpu_file, tracing_reader_cpumask))
- cpumask_set_cpu(cpu_file, tracing_reader_cpumask);
- else {
- ret = -EBUSY;
- goto out;
- }
- }
-
/* create a buffer to store the information to pass to userspace */
iter = kzalloc(sizeof(*iter), GFP_KERNEL);
if (!iter) {
@@ -2912,12 +2975,6 @@ static int tracing_release_pipe(struct inode *inode, struct file *file)
mutex_lock(&trace_types_lock);
- if (iter->cpu_file == TRACE_PIPE_ALL_CPU)
- cpumask_clear(tracing_reader_cpumask);
- else
- cpumask_clear_cpu(iter->cpu_file, tracing_reader_cpumask);
-
-
if (iter->trace->pipe_close)
iter->trace->pipe_close(iter);
@@ -3079,6 +3136,7 @@ waitagain:
iter->pos = -1;
trace_event_read_lock();
+ trace_access_lock(iter->cpu_file);
while (find_next_entry_inc(iter) != NULL) {
enum print_line_t ret;
int len = iter->seq.len;
@@ -3095,6 +3153,7 @@ waitagain:
if (iter->seq.len >= cnt)
break;
}
+ trace_access_unlock(iter->cpu_file);
trace_event_read_unlock();
/* Now copy what we have to the user */
@@ -3220,6 +3279,7 @@ static ssize_t tracing_splice_read_pipe(struct file *filp,
}
trace_event_read_lock();
+ trace_access_lock(iter->cpu_file);
/* Fill as many pages as possible. */
for (i = 0, rem = len; i < PIPE_BUFFERS && rem; i++) {
@@ -3243,6 +3303,7 @@ static ssize_t tracing_splice_read_pipe(struct file *filp,
trace_seq_init(&iter->seq);
}
+ trace_access_unlock(iter->cpu_file);
trace_event_read_unlock();
mutex_unlock(&iter->mutex);
@@ -3544,10 +3605,12 @@ tracing_buffers_read(struct file *filp, char __user *ubuf,
info->read = 0;
+ trace_access_lock(info->cpu);
ret = ring_buffer_read_page(info->tr->buffer,
&info->spare,
count,
info->cpu, 0);
+ trace_access_unlock(info->cpu);
if (ret < 0)
return 0;
@@ -3675,6 +3738,7 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos,
len &= PAGE_MASK;
}
+ trace_access_lock(info->cpu);
entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
for (i = 0; i < PIPE_BUFFERS && len && entries; i++, len -= PAGE_SIZE) {
@@ -3722,6 +3786,7 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos,
entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
}
+ trace_access_unlock(info->cpu);
spd.nr_pages = i;
/* did we read anything? */
@@ -4158,6 +4223,8 @@ static __init int tracer_init_debugfs(void)
struct dentry *d_tracer;
int cpu;
+ trace_access_lock_init();
+
d_tracer = tracing_init_dentry();
trace_create_file("tracing_enabled", 0644, d_tracer,
@@ -4392,9 +4459,6 @@ __init static int tracer_alloc_buffers(void)
if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
goto out_free_buffer_mask;
- if (!zalloc_cpumask_var(&tracing_reader_cpumask, GFP_KERNEL))
- goto out_free_tracing_cpumask;
-
/* To save memory, keep the ring buffer size to its minimum */
if (ring_buffer_expanded)
ring_buf_size = trace_buf_size;
@@ -4452,8 +4516,6 @@ __init static int tracer_alloc_buffers(void)
return 0;
out_free_cpumask:
- free_cpumask_var(tracing_reader_cpumask);
-out_free_tracing_cpumask:
free_cpumask_var(tracing_cpumask);
out_free_buffer_mask:
free_cpumask_var(tracing_buffer_mask);
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index 4df6a77eb196..fd05bcaf91b0 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -497,6 +497,7 @@ trace_print_graph_duration(unsigned long long duration, struct trace_seq *s);
#ifdef CONFIG_DYNAMIC_FTRACE
/* TODO: make this variable */
#define FTRACE_GRAPH_MAX_FUNCS 32
+extern int ftrace_graph_filter_enabled;
extern int ftrace_graph_count;
extern unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS];
@@ -504,7 +505,7 @@ static inline int ftrace_graph_addr(unsigned long addr)
{
int i;
- if (!ftrace_graph_count || test_tsk_trace_graph(current))
+ if (!ftrace_graph_filter_enabled)
return 1;
for (i = 0; i < ftrace_graph_count; i++) {
@@ -791,7 +792,8 @@ extern const char *__stop___trace_bprintk_fmt[];
#undef FTRACE_ENTRY
#define FTRACE_ENTRY(call, struct_name, id, tstruct, print) \
- extern struct ftrace_event_call event_##call;
+ extern struct ftrace_event_call \
+ __attribute__((__aligned__(4))) event_##call;
#undef FTRACE_ENTRY_DUP
#define FTRACE_ENTRY_DUP(call, struct_name, id, tstruct, print) \
FTRACE_ENTRY(call, struct_name, id, PARAMS(tstruct), PARAMS(print))
diff --git a/kernel/trace/trace_branch.c b/kernel/trace/trace_branch.c
index 4a194f08f88c..b9bc4d470177 100644
--- a/kernel/trace/trace_branch.c
+++ b/kernel/trace/trace_branch.c
@@ -307,8 +307,23 @@ static int annotated_branch_stat_cmp(void *p1, void *p2)
return -1;
if (percent_a > percent_b)
return 1;
- else
- return 0;
+
+ if (a->incorrect < b->incorrect)
+ return -1;
+ if (a->incorrect > b->incorrect)
+ return 1;
+
+ /*
+ * Since the above shows worse (incorrect) cases
+ * first, we continue that by showing best (correct)
+ * cases last.
+ */
+ if (a->correct > b->correct)
+ return -1;
+ if (a->correct < b->correct)
+ return 1;
+
+ return 0;
}
static struct tracer_stat annotated_branch_stats = {
diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c
index 189b09baf4fb..3f972ad98d04 100644
--- a/kernel/trace/trace_events.c
+++ b/kernel/trace/trace_events.c
@@ -60,10 +60,8 @@ int trace_define_field(struct ftrace_event_call *call, const char *type,
return 0;
err:
- if (field) {
+ if (field)
kfree(field->name);
- kfree(field->type);
- }
kfree(field);
return -ENOMEM;
@@ -520,41 +518,16 @@ out:
return ret;
}
-extern char *__bad_type_size(void);
-
-#undef FIELD
-#define FIELD(type, name) \
- sizeof(type) != sizeof(field.name) ? __bad_type_size() : \
- #type, "common_" #name, offsetof(typeof(field), name), \
- sizeof(field.name), is_signed_type(type)
-
-static int trace_write_header(struct trace_seq *s)
-{
- struct trace_entry field;
-
- /* struct trace_entry */
- return trace_seq_printf(s,
- "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\tsigned:%u;\n"
- "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\tsigned:%u;\n"
- "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\tsigned:%u;\n"
- "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\tsigned:%u;\n"
- "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\tsigned:%u;\n"
- "\n",
- FIELD(unsigned short, type),
- FIELD(unsigned char, flags),
- FIELD(unsigned char, preempt_count),
- FIELD(int, pid),
- FIELD(int, lock_depth));
-}
-
static ssize_t
event_format_read(struct file *filp, char __user *ubuf, size_t cnt,
loff_t *ppos)
{
struct ftrace_event_call *call = filp->private_data;
+ struct ftrace_event_field *field;
struct trace_seq *s;
+ int common_field_count = 5;
char *buf;
- int r;
+ int r = 0;
if (*ppos)
return 0;
@@ -565,14 +538,48 @@ event_format_read(struct file *filp, char __user *ubuf, size_t cnt,
trace_seq_init(s);
- /* If any of the first writes fail, so will the show_format. */
-
trace_seq_printf(s, "name: %s\n", call->name);
trace_seq_printf(s, "ID: %d\n", call->id);
trace_seq_printf(s, "format:\n");
- trace_write_header(s);
- r = call->show_format(call, s);
+ list_for_each_entry_reverse(field, &call->fields, link) {
+ /*
+ * Smartly shows the array type(except dynamic array).
+ * Normal:
+ * field:TYPE VAR
+ * If TYPE := TYPE[LEN], it is shown:
+ * field:TYPE VAR[LEN]
+ */
+ const char *array_descriptor = strchr(field->type, '[');
+
+ if (!strncmp(field->type, "__data_loc", 10))
+ array_descriptor = NULL;
+
+ if (!array_descriptor) {
+ r = trace_seq_printf(s, "\tfield:%s %s;\toffset:%u;"
+ "\tsize:%u;\tsigned:%d;\n",
+ field->type, field->name, field->offset,
+ field->size, !!field->is_signed);
+ } else {
+ r = trace_seq_printf(s, "\tfield:%.*s %s%s;\toffset:%u;"
+ "\tsize:%u;\tsigned:%d;\n",
+ (int)(array_descriptor - field->type),
+ field->type, field->name,
+ array_descriptor, field->offset,
+ field->size, !!field->is_signed);
+ }
+
+ if (--common_field_count == 0)
+ r = trace_seq_printf(s, "\n");
+
+ if (!r)
+ break;
+ }
+
+ if (r)
+ r = trace_seq_printf(s, "\nprint fmt: %s\n",
+ call->print_fmt);
+
if (!r) {
/*
* ug! The format output is bigger than a PAGE!!
@@ -948,10 +955,6 @@ event_create_dir(struct ftrace_event_call *call, struct dentry *d_events,
filter);
}
- /* A trace may not want to export its format */
- if (!call->show_format)
- return 0;
-
trace_create_file("format", 0444, call->dir, call,
format);
diff --git a/kernel/trace/trace_export.c b/kernel/trace/trace_export.c
index d4fa5dc1ee4e..e091f64ba6ce 100644
--- a/kernel/trace/trace_export.c
+++ b/kernel/trace/trace_export.c
@@ -62,78 +62,6 @@ static void __always_unused ____ftrace_check_##name(void) \
#include "trace_entries.h"
-
-#undef __field
-#define __field(type, item) \
- ret = trace_seq_printf(s, "\tfield:" #type " " #item ";\t" \
- "offset:%zu;\tsize:%zu;\tsigned:%u;\n", \
- offsetof(typeof(field), item), \
- sizeof(field.item), is_signed_type(type)); \
- if (!ret) \
- return 0;
-
-#undef __field_desc
-#define __field_desc(type, container, item) \
- ret = trace_seq_printf(s, "\tfield:" #type " " #item ";\t" \
- "offset:%zu;\tsize:%zu;\tsigned:%u;\n", \
- offsetof(typeof(field), container.item), \
- sizeof(field.container.item), \
- is_signed_type(type)); \
- if (!ret) \
- return 0;
-
-#undef __array
-#define __array(type, item, len) \
- ret = trace_seq_printf(s, "\tfield:" #type " " #item "[" #len "];\t" \
- "offset:%zu;\tsize:%zu;\tsigned:%u;\n", \
- offsetof(typeof(field), item), \
- sizeof(field.item), is_signed_type(type)); \
- if (!ret) \
- return 0;
-
-#undef __array_desc
-#define __array_desc(type, container, item, len) \
- ret = trace_seq_printf(s, "\tfield:" #type " " #item "[" #len "];\t" \
- "offset:%zu;\tsize:%zu;\tsigned:%u;\n", \
- offsetof(typeof(field), container.item), \
- sizeof(field.container.item), \
- is_signed_type(type)); \
- if (!ret) \
- return 0;
-
-#undef __dynamic_array
-#define __dynamic_array(type, item) \
- ret = trace_seq_printf(s, "\tfield:" #type " " #item ";\t" \
- "offset:%zu;\tsize:0;\tsigned:%u;\n", \
- offsetof(typeof(field), item), \
- is_signed_type(type)); \
- if (!ret) \
- return 0;
-
-#undef F_printk
-#define F_printk(fmt, args...) "%s, %s\n", #fmt, __stringify(args)
-
-#undef __entry
-#define __entry REC
-
-#undef FTRACE_ENTRY
-#define FTRACE_ENTRY(name, struct_name, id, tstruct, print) \
-static int \
-ftrace_format_##name(struct ftrace_event_call *unused, \
- struct trace_seq *s) \
-{ \
- struct struct_name field __attribute__((unused)); \
- int ret = 0; \
- \
- tstruct; \
- \
- trace_seq_printf(s, "\nprint fmt: " print); \
- \
- return ret; \
-}
-
-#include "trace_entries.h"
-
#undef __field
#define __field(type, item) \
ret = trace_define_field(event_call, #type, #item, \
@@ -175,7 +103,12 @@ ftrace_format_##name(struct ftrace_event_call *unused, \
return ret;
#undef __dynamic_array
-#define __dynamic_array(type, item)
+#define __dynamic_array(type, item) \
+ ret = trace_define_field(event_call, #type, #item, \
+ offsetof(typeof(field), item), \
+ 0, is_signed_type(type), FILTER_OTHER);\
+ if (ret) \
+ return ret;
#undef FTRACE_ENTRY
#define FTRACE_ENTRY(name, struct_name, id, tstruct, print) \
@@ -198,6 +131,9 @@ static int ftrace_raw_init_event(struct ftrace_event_call *call)
return 0;
}
+#undef __entry
+#define __entry REC
+
#undef __field
#define __field(type, item)
@@ -213,6 +149,9 @@ static int ftrace_raw_init_event(struct ftrace_event_call *call)
#undef __dynamic_array
#define __dynamic_array(type, item)
+#undef F_printk
+#define F_printk(fmt, args...) #fmt ", " __stringify(args)
+
#undef FTRACE_ENTRY
#define FTRACE_ENTRY(call, struct_name, type, tstruct, print) \
\
@@ -223,7 +162,7 @@ __attribute__((section("_ftrace_events"))) event_##call = { \
.id = type, \
.system = __stringify(TRACE_SYSTEM), \
.raw_init = ftrace_raw_init_event, \
- .show_format = ftrace_format_##call, \
+ .print_fmt = print, \
.define_fields = ftrace_define_fields_##call, \
}; \
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
index b1342c5d37cf..e998a824e9db 100644
--- a/kernel/trace/trace_functions_graph.c
+++ b/kernel/trace/trace_functions_graph.c
@@ -18,6 +18,7 @@ struct fgraph_cpu_data {
pid_t last_pid;
int depth;
int ignore;
+ unsigned long enter_funcs[FTRACE_RETFUNC_DEPTH];
};
struct fgraph_data {
@@ -212,13 +213,11 @@ int trace_graph_entry(struct ftrace_graph_ent *trace)
int cpu;
int pc;
- if (unlikely(!tr))
- return 0;
-
if (!ftrace_trace_task(current))
return 0;
- if (!ftrace_graph_addr(trace->func))
+ /* trace it when it is-nested-in or is a function enabled. */
+ if (!(trace->depth || ftrace_graph_addr(trace->func)))
return 0;
local_irq_save(flags);
@@ -231,9 +230,6 @@ int trace_graph_entry(struct ftrace_graph_ent *trace)
} else {
ret = 0;
}
- /* Only do the atomic if it is not already set */
- if (!test_tsk_trace_graph(current))
- set_tsk_trace_graph(current);
atomic_dec(&data->disabled);
local_irq_restore(flags);
@@ -281,17 +277,24 @@ void trace_graph_return(struct ftrace_graph_ret *trace)
pc = preempt_count();
__trace_graph_return(tr, trace, flags, pc);
}
- if (!trace->depth)
- clear_tsk_trace_graph(current);
atomic_dec(&data->disabled);
local_irq_restore(flags);
}
+void set_graph_array(struct trace_array *tr)
+{
+ graph_array = tr;
+
+ /* Make graph_array visible before we start tracing */
+
+ smp_mb();
+}
+
static int graph_trace_init(struct trace_array *tr)
{
int ret;
- graph_array = tr;
+ set_graph_array(tr);
ret = register_ftrace_graph(&trace_graph_return,
&trace_graph_entry);
if (ret)
@@ -301,11 +304,6 @@ static int graph_trace_init(struct trace_array *tr)
return 0;
}
-void set_graph_array(struct trace_array *tr)
-{
- graph_array = tr;
-}
-
static void graph_trace_reset(struct trace_array *tr)
{
tracing_stop_cmdline_record();
@@ -673,15 +671,21 @@ print_graph_entry_leaf(struct trace_iterator *iter,
duration = graph_ret->rettime - graph_ret->calltime;
if (data) {
+ struct fgraph_cpu_data *cpu_data;
int cpu = iter->cpu;
- int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth);
+
+ cpu_data = per_cpu_ptr(data->cpu_data, cpu);
/*
* Comments display at + 1 to depth. Since
* this is a leaf function, keep the comments
* equal to this depth.
*/
- *depth = call->depth - 1;
+ cpu_data->depth = call->depth - 1;
+
+ /* No need to keep this function around for this depth */
+ if (call->depth < FTRACE_RETFUNC_DEPTH)
+ cpu_data->enter_funcs[call->depth] = 0;
}
/* Overhead */
@@ -721,10 +725,15 @@ print_graph_entry_nested(struct trace_iterator *iter,
int i;
if (data) {
+ struct fgraph_cpu_data *cpu_data;
int cpu = iter->cpu;
- int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth);
- *depth = call->depth;
+ cpu_data = per_cpu_ptr(data->cpu_data, cpu);
+ cpu_data->depth = call->depth;
+
+ /* Save this function pointer to see if the exit matches */
+ if (call->depth < FTRACE_RETFUNC_DEPTH)
+ cpu_data->enter_funcs[call->depth] = call->func;
}
/* No overhead */
@@ -854,19 +863,28 @@ print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s,
struct fgraph_data *data = iter->private;
pid_t pid = ent->pid;
int cpu = iter->cpu;
+ int func_match = 1;
int ret;
int i;
if (data) {
+ struct fgraph_cpu_data *cpu_data;
int cpu = iter->cpu;
- int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth);
+
+ cpu_data = per_cpu_ptr(data->cpu_data, cpu);
/*
* Comments display at + 1 to depth. This is the
* return from a function, we now want the comments
* to display at the same level of the bracket.
*/
- *depth = trace->depth - 1;
+ cpu_data->depth = trace->depth - 1;
+
+ if (trace->depth < FTRACE_RETFUNC_DEPTH) {
+ if (cpu_data->enter_funcs[trace->depth] != trace->func)
+ func_match = 0;
+ cpu_data->enter_funcs[trace->depth] = 0;
+ }
}
if (print_graph_prologue(iter, s, 0, 0))
@@ -891,9 +909,21 @@ print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s,
return TRACE_TYPE_PARTIAL_LINE;
}
- ret = trace_seq_printf(s, "}\n");
- if (!ret)
- return TRACE_TYPE_PARTIAL_LINE;
+ /*
+ * If the return function does not have a matching entry,
+ * then the entry was lost. Instead of just printing
+ * the '}' and letting the user guess what function this
+ * belongs to, write out the function name.
+ */
+ if (func_match) {
+ ret = trace_seq_printf(s, "}\n");
+ if (!ret)
+ return TRACE_TYPE_PARTIAL_LINE;
+ } else {
+ ret = trace_seq_printf(s, "} (%ps)\n", (void *)trace->func);
+ if (!ret)
+ return TRACE_TYPE_PARTIAL_LINE;
+ }
/* Overrun */
if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERRUN) {
diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c
index 50b1b8239806..465b36bef4ca 100644
--- a/kernel/trace/trace_kprobe.c
+++ b/kernel/trace/trace_kprobe.c
@@ -651,12 +651,12 @@ static int create_trace_probe(int argc, char **argv)
event = strchr(group, '/') + 1;
event[-1] = '\0';
if (strlen(group) == 0) {
- pr_info("Group name is not specifiled\n");
+ pr_info("Group name is not specified\n");
return -EINVAL;
}
}
if (strlen(event) == 0) {
- pr_info("Event name is not specifiled\n");
+ pr_info("Event name is not specified\n");
return -EINVAL;
}
}
@@ -1174,80 +1174,60 @@ static int kretprobe_event_define_fields(struct ftrace_event_call *event_call)
return 0;
}
-static int __probe_event_show_format(struct trace_seq *s,
- struct trace_probe *tp, const char *fmt,
- const char *arg)
+static int __set_print_fmt(struct trace_probe *tp, char *buf, int len)
{
int i;
+ int pos = 0;
- /* Show format */
- if (!trace_seq_printf(s, "\nprint fmt: \"%s", fmt))
- return 0;
+ const char *fmt, *arg;
- for (i = 0; i < tp->nr_args; i++)
- if (!trace_seq_printf(s, " %s=%%lx", tp->args[i].name))
- return 0;
+ if (!probe_is_return(tp)) {
+ fmt = "(%lx)";
+ arg = "REC->" FIELD_STRING_IP;
+ } else {
+ fmt = "(%lx <- %lx)";
+ arg = "REC->" FIELD_STRING_FUNC ", REC->" FIELD_STRING_RETIP;
+ }
- if (!trace_seq_printf(s, "\", %s", arg))
- return 0;
+ /* When len=0, we just calculate the needed length */
+#define LEN_OR_ZERO (len ? len - pos : 0)
- for (i = 0; i < tp->nr_args; i++)
- if (!trace_seq_printf(s, ", REC->%s", tp->args[i].name))
- return 0;
+ pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", fmt);
- return trace_seq_puts(s, "\n");
-}
+ for (i = 0; i < tp->nr_args; i++) {
+ pos += snprintf(buf + pos, LEN_OR_ZERO, " %s=%%lx",
+ tp->args[i].name);
+ }
-#undef SHOW_FIELD
-#define SHOW_FIELD(type, item, name) \
- do { \
- ret = trace_seq_printf(s, "\tfield:" #type " %s;\t" \
- "offset:%u;\tsize:%u;\tsigned:%d;\n", name,\
- (unsigned int)offsetof(typeof(field), item),\
- (unsigned int)sizeof(type), \
- is_signed_type(type)); \
- if (!ret) \
- return 0; \
- } while (0)
+ pos += snprintf(buf + pos, LEN_OR_ZERO, "\", %s", arg);
-static int kprobe_event_show_format(struct ftrace_event_call *call,
- struct trace_seq *s)
-{
- struct kprobe_trace_entry field __attribute__((unused));
- int ret, i;
- struct trace_probe *tp = (struct trace_probe *)call->data;
-
- SHOW_FIELD(unsigned long, ip, FIELD_STRING_IP);
- SHOW_FIELD(int, nargs, FIELD_STRING_NARGS);
+ for (i = 0; i < tp->nr_args; i++) {
+ pos += snprintf(buf + pos, LEN_OR_ZERO, ", REC->%s",
+ tp->args[i].name);
+ }
- /* Show fields */
- for (i = 0; i < tp->nr_args; i++)
- SHOW_FIELD(unsigned long, args[i], tp->args[i].name);
- trace_seq_puts(s, "\n");
+#undef LEN_OR_ZERO
- return __probe_event_show_format(s, tp, "(%lx)",
- "REC->" FIELD_STRING_IP);
+ /* return the length of print_fmt */
+ return pos;
}
-static int kretprobe_event_show_format(struct ftrace_event_call *call,
- struct trace_seq *s)
+static int set_print_fmt(struct trace_probe *tp)
{
- struct kretprobe_trace_entry field __attribute__((unused));
- int ret, i;
- struct trace_probe *tp = (struct trace_probe *)call->data;
+ int len;
+ char *print_fmt;
- SHOW_FIELD(unsigned long, func, FIELD_STRING_FUNC);
- SHOW_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP);
- SHOW_FIELD(int, nargs, FIELD_STRING_NARGS);
+ /* First: called with 0 length to calculate the needed length */
+ len = __set_print_fmt(tp, NULL, 0);
+ print_fmt = kmalloc(len + 1, GFP_KERNEL);
+ if (!print_fmt)
+ return -ENOMEM;
- /* Show fields */
- for (i = 0; i < tp->nr_args; i++)
- SHOW_FIELD(unsigned long, args[i], tp->args[i].name);
- trace_seq_puts(s, "\n");
+ /* Second: actually write the @print_fmt */
+ __set_print_fmt(tp, print_fmt, len + 1);
+ tp->call.print_fmt = print_fmt;
- return __probe_event_show_format(s, tp, "(%lx <- %lx)",
- "REC->" FIELD_STRING_FUNC
- ", REC->" FIELD_STRING_RETIP);
+ return 0;
}
#ifdef CONFIG_EVENT_PROFILE
@@ -1448,18 +1428,20 @@ static int register_probe_event(struct trace_probe *tp)
if (probe_is_return(tp)) {
tp->event.trace = print_kretprobe_event;
call->raw_init = probe_event_raw_init;
- call->show_format = kretprobe_event_show_format;
call->define_fields = kretprobe_event_define_fields;
} else {
tp->event.trace = print_kprobe_event;
call->raw_init = probe_event_raw_init;
- call->show_format = kprobe_event_show_format;
call->define_fields = kprobe_event_define_fields;
}
+ if (set_print_fmt(tp) < 0)
+ return -ENOMEM;
call->event = &tp->event;
call->id = register_ftrace_event(&tp->event);
- if (!call->id)
+ if (!call->id) {
+ kfree(call->print_fmt);
return -ENODEV;
+ }
call->enabled = 0;
call->regfunc = probe_event_enable;
call->unregfunc = probe_event_disable;
@@ -1472,6 +1454,7 @@ static int register_probe_event(struct trace_probe *tp)
ret = trace_add_event_call(call);
if (ret) {
pr_info("Failed to register kprobe event: %s\n", call->name);
+ kfree(call->print_fmt);
unregister_ftrace_event(&tp->event);
}
return ret;
@@ -1481,6 +1464,7 @@ static void unregister_probe_event(struct trace_probe *tp)
{
/* tp->event is unregistered in trace_remove_event_call() */
trace_remove_event_call(&tp->call);
+ kfree(tp->call.print_fmt);
}
/* Make a debugfs interface for controling probe points */
diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c
index 75289f372dd2..a1834dda85f4 100644
--- a/kernel/trace/trace_syscalls.c
+++ b/kernel/trace/trace_syscalls.c
@@ -143,70 +143,65 @@ extern char *__bad_type_size(void);
#type, #name, offsetof(typeof(trace), name), \
sizeof(trace.name), is_signed_type(type)
-int syscall_enter_format(struct ftrace_event_call *call, struct trace_seq *s)
+static
+int __set_enter_print_fmt(struct syscall_metadata *entry, char *buf, int len)
{
int i;
- int ret;
- struct syscall_metadata *entry = call->data;
- struct syscall_trace_enter trace;
- int offset = offsetof(struct syscall_trace_enter, args);
+ int pos = 0;
- ret = trace_seq_printf(s, "\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
- "\tsigned:%u;\n",
- SYSCALL_FIELD(int, nr));
- if (!ret)
- return 0;
+ /* When len=0, we just calculate the needed length */
+#define LEN_OR_ZERO (len ? len - pos : 0)
+ pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
for (i = 0; i < entry->nb_args; i++) {
- ret = trace_seq_printf(s, "\tfield:%s %s;", entry->types[i],
- entry->args[i]);
- if (!ret)
- return 0;
- ret = trace_seq_printf(s, "\toffset:%d;\tsize:%zu;"
- "\tsigned:%u;\n", offset,
- sizeof(unsigned long),
- is_signed_type(unsigned long));
- if (!ret)
- return 0;
- offset += sizeof(unsigned long);
+ pos += snprintf(buf + pos, LEN_OR_ZERO, "%s: 0x%%0%zulx%s",
+ entry->args[i], sizeof(unsigned long),
+ i == entry->nb_args - 1 ? "" : ", ");
}
+ pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
- trace_seq_puts(s, "\nprint fmt: \"");
for (i = 0; i < entry->nb_args; i++) {
- ret = trace_seq_printf(s, "%s: 0x%%0%zulx%s", entry->args[i],
- sizeof(unsigned long),
- i == entry->nb_args - 1 ? "" : ", ");
- if (!ret)
- return 0;
+ pos += snprintf(buf + pos, LEN_OR_ZERO,
+ ", ((unsigned long)(REC->%s))", entry->args[i]);
}
- trace_seq_putc(s, '"');
- for (i = 0; i < entry->nb_args; i++) {
- ret = trace_seq_printf(s, ", ((unsigned long)(REC->%s))",
- entry->args[i]);
- if (!ret)
- return 0;
- }
+#undef LEN_OR_ZERO
- return trace_seq_putc(s, '\n');
+ /* return the length of print_fmt */
+ return pos;
}
-int syscall_exit_format(struct ftrace_event_call *call, struct trace_seq *s)
+static int set_syscall_print_fmt(struct ftrace_event_call *call)
{
- int ret;
- struct syscall_trace_exit trace;
+ char *print_fmt;
+ int len;
+ struct syscall_metadata *entry = call->data;
- ret = trace_seq_printf(s,
- "\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
- "\tsigned:%u;\n"
- "\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
- "\tsigned:%u;\n",
- SYSCALL_FIELD(int, nr),
- SYSCALL_FIELD(long, ret));
- if (!ret)
+ if (entry->enter_event != call) {
+ call->print_fmt = "\"0x%lx\", REC->ret";
return 0;
+ }
- return trace_seq_printf(s, "\nprint fmt: \"0x%%lx\", REC->ret\n");
+ /* First: called with 0 length to calculate the needed length */
+ len = __set_enter_print_fmt(entry, NULL, 0);
+
+ print_fmt = kmalloc(len + 1, GFP_KERNEL);
+ if (!print_fmt)
+ return -ENOMEM;
+
+ /* Second: actually write the @print_fmt */
+ __set_enter_print_fmt(entry, print_fmt, len + 1);
+ call->print_fmt = print_fmt;
+
+ return 0;
+}
+
+static void free_syscall_print_fmt(struct ftrace_event_call *call)
+{
+ struct syscall_metadata *entry = call->data;
+
+ if (entry->enter_event == call)
+ kfree(call->print_fmt);
}
int syscall_enter_define_fields(struct ftrace_event_call *call)
@@ -386,12 +381,22 @@ int init_syscall_trace(struct ftrace_event_call *call)
{
int id;
- id = register_ftrace_event(call->event);
- if (!id)
- return -ENODEV;
- call->id = id;
- INIT_LIST_HEAD(&call->fields);
- return 0;
+ if (set_syscall_print_fmt(call) < 0)
+ return -ENOMEM;
+
+ id = trace_event_raw_init(call);
+
+ if (id < 0) {
+ free_syscall_print_fmt(call);
+ return id;
+ }
+
+ return id;
+}
+
+unsigned long __init arch_syscall_addr(int nr)
+{
+ return (unsigned long)sys_call_table[nr];
}
int __init init_ftrace_syscalls(void)
@@ -603,7 +608,7 @@ int prof_sysexit_enable(struct ftrace_event_call *call)
ret = register_trace_sys_exit(prof_syscall_exit);
if (ret) {
pr_info("event trace: Could not activate"
- "syscall entry trace point");
+ "syscall exit trace point");
} else {
set_bit(num, enabled_prof_exit_syscalls);
sys_prof_refcount_exit++;
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-02 22:39:08 +0000