perf_events: Undo some recursion damage
Make perf_swevent_get_recursion_context return a context number and disable preemption. This could be used to remove the IRQ disable from the trace bit and index the per-cpu buffer with. Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Frederic Weisbecker <fweisbec@gmail.com> Cc: Paul Mackerras <paulus@samba.org> LKML-Reference: <20091123103819.993226816@chello.nl> Signed-off-by: Ingo Molnar <mingo@elte.hu>
This commit is contained in:
parent
f67218c3e9
commit
4ed7c92d68
5 changed files with 61 additions and 63 deletions
|
@ -874,8 +874,8 @@ extern int perf_output_begin(struct perf_output_handle *handle,
|
|||
extern void perf_output_end(struct perf_output_handle *handle);
|
||||
extern void perf_output_copy(struct perf_output_handle *handle,
|
||||
const void *buf, unsigned int len);
|
||||
extern int perf_swevent_get_recursion_context(int **recursion);
|
||||
extern void perf_swevent_put_recursion_context(int *recursion);
|
||||
extern int perf_swevent_get_recursion_context(void);
|
||||
extern void perf_swevent_put_recursion_context(int rctx);
|
||||
#else
|
||||
static inline void
|
||||
perf_event_task_sched_in(struct task_struct *task, int cpu) { }
|
||||
|
@ -904,8 +904,8 @@ static inline void perf_event_mmap(struct vm_area_struct *vma) { }
|
|||
static inline void perf_event_comm(struct task_struct *tsk) { }
|
||||
static inline void perf_event_fork(struct task_struct *tsk) { }
|
||||
static inline void perf_event_init(void) { }
|
||||
static int perf_swevent_get_recursion_context(int **recursion) { return -1; }
|
||||
static void perf_swevent_put_recursion_context(int *recursion) { }
|
||||
static inline int perf_swevent_get_recursion_context(void) { return -1; }
|
||||
static inline void perf_swevent_put_recursion_context(int rctx) { }
|
||||
|
||||
#endif
|
||||
|
||||
|
|
|
@ -724,8 +724,8 @@ __attribute__((section("_ftrace_events"))) event_##call = { \
|
|||
static void ftrace_profile_##call(proto) \
|
||||
{ \
|
||||
struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\
|
||||
extern int perf_swevent_get_recursion_context(int **recursion); \
|
||||
extern void perf_swevent_put_recursion_context(int *recursion); \
|
||||
extern int perf_swevent_get_recursion_context(void); \
|
||||
extern void perf_swevent_put_recursion_context(int rctx); \
|
||||
struct ftrace_event_call *event_call = &event_##call; \
|
||||
extern void perf_tp_event(int, u64, u64, void *, int); \
|
||||
struct ftrace_raw_##call *entry; \
|
||||
|
@ -736,8 +736,8 @@ static void ftrace_profile_##call(proto) \
|
|||
int __data_size; \
|
||||
char *trace_buf; \
|
||||
char *raw_data; \
|
||||
int *recursion; \
|
||||
int __cpu; \
|
||||
int rctx; \
|
||||
int pc; \
|
||||
\
|
||||
pc = preempt_count(); \
|
||||
|
@ -753,8 +753,9 @@ static void ftrace_profile_##call(proto) \
|
|||
\
|
||||
local_irq_save(irq_flags); \
|
||||
\
|
||||
if (perf_swevent_get_recursion_context(&recursion)) \
|
||||
goto end_recursion; \
|
||||
rctx = perf_swevent_get_recursion_context(); \
|
||||
if (rctx < 0) \
|
||||
goto end_recursion; \
|
||||
\
|
||||
__cpu = smp_processor_id(); \
|
||||
\
|
||||
|
@ -781,9 +782,9 @@ static void ftrace_profile_##call(proto) \
|
|||
perf_tp_event(event_call->id, __addr, __count, entry, \
|
||||
__entry_size); \
|
||||
\
|
||||
end: \
|
||||
perf_swevent_put_recursion_context(recursion); \
|
||||
end_recursion: \
|
||||
end: \
|
||||
perf_swevent_put_recursion_context(rctx); \
|
||||
end_recursion: \
|
||||
local_irq_restore(irq_flags); \
|
||||
\
|
||||
}
|
||||
|
|
|
@ -3869,45 +3869,50 @@ static void perf_swevent_ctx_event(struct perf_event_context *ctx,
|
|||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Must be called with preemption disabled
|
||||
*/
|
||||
int perf_swevent_get_recursion_context(int **recursion)
|
||||
int perf_swevent_get_recursion_context(void)
|
||||
{
|
||||
struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
|
||||
struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
|
||||
int rctx;
|
||||
|
||||
if (in_nmi())
|
||||
*recursion = &cpuctx->recursion[3];
|
||||
rctx = 3;
|
||||
else if (in_irq())
|
||||
*recursion = &cpuctx->recursion[2];
|
||||
rctx = 2;
|
||||
else if (in_softirq())
|
||||
*recursion = &cpuctx->recursion[1];
|
||||
rctx = 1;
|
||||
else
|
||||
*recursion = &cpuctx->recursion[0];
|
||||
rctx = 0;
|
||||
|
||||
if (**recursion)
|
||||
if (cpuctx->recursion[rctx]) {
|
||||
put_cpu_var(perf_cpu_context);
|
||||
return -1;
|
||||
}
|
||||
|
||||
(**recursion)++;
|
||||
cpuctx->recursion[rctx]++;
|
||||
barrier();
|
||||
|
||||
return 0;
|
||||
return rctx;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
|
||||
|
||||
void perf_swevent_put_recursion_context(int *recursion)
|
||||
void perf_swevent_put_recursion_context(int rctx)
|
||||
{
|
||||
(*recursion)--;
|
||||
struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
|
||||
barrier();
|
||||
cpuctx->recursion[rctx]++;
|
||||
put_cpu_var(perf_cpu_context);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(perf_swevent_put_recursion_context);
|
||||
|
||||
static void __do_perf_sw_event(enum perf_type_id type, u32 event_id,
|
||||
u64 nr, int nmi,
|
||||
struct perf_sample_data *data,
|
||||
struct pt_regs *regs)
|
||||
static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
|
||||
u64 nr, int nmi,
|
||||
struct perf_sample_data *data,
|
||||
struct pt_regs *regs)
|
||||
{
|
||||
struct perf_cpu_context *cpuctx;
|
||||
struct perf_event_context *ctx;
|
||||
struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
|
||||
|
||||
cpuctx = &__get_cpu_var(perf_cpu_context);
|
||||
rcu_read_lock();
|
||||
perf_swevent_ctx_event(&cpuctx->ctx, type, event_id,
|
||||
nr, nmi, data, regs);
|
||||
|
@ -3921,34 +3926,22 @@ static void __do_perf_sw_event(enum perf_type_id type, u32 event_id,
|
|||
rcu_read_unlock();
|
||||
}
|
||||
|
||||
static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
|
||||
u64 nr, int nmi,
|
||||
struct perf_sample_data *data,
|
||||
struct pt_regs *regs)
|
||||
{
|
||||
int *recursion;
|
||||
|
||||
preempt_disable();
|
||||
|
||||
if (perf_swevent_get_recursion_context(&recursion))
|
||||
goto out;
|
||||
|
||||
__do_perf_sw_event(type, event_id, nr, nmi, data, regs);
|
||||
|
||||
perf_swevent_put_recursion_context(recursion);
|
||||
out:
|
||||
preempt_enable();
|
||||
}
|
||||
|
||||
void __perf_sw_event(u32 event_id, u64 nr, int nmi,
|
||||
struct pt_regs *regs, u64 addr)
|
||||
{
|
||||
struct perf_sample_data data;
|
||||
int rctx;
|
||||
|
||||
rctx = perf_swevent_get_recursion_context();
|
||||
if (rctx < 0)
|
||||
return;
|
||||
|
||||
data.addr = addr;
|
||||
data.raw = NULL;
|
||||
|
||||
do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi, &data, regs);
|
||||
|
||||
perf_swevent_put_recursion_context(rctx);
|
||||
}
|
||||
|
||||
static void perf_swevent_read(struct perf_event *event)
|
||||
|
@ -4172,7 +4165,7 @@ void perf_tp_event(int event_id, u64 addr, u64 count, void *record,
|
|||
regs = task_pt_regs(current);
|
||||
|
||||
/* Trace events already protected against recursion */
|
||||
__do_perf_sw_event(PERF_TYPE_TRACEPOINT, event_id, count, 1,
|
||||
do_perf_sw_event(PERF_TYPE_TRACEPOINT, event_id, count, 1,
|
||||
&data, regs);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(perf_tp_event);
|
||||
|
|
|
@ -1213,7 +1213,7 @@ static __kprobes int kprobe_profile_func(struct kprobe *kp,
|
|||
unsigned long irq_flags;
|
||||
char *trace_buf;
|
||||
char *raw_data;
|
||||
int *recursion;
|
||||
int rctx;
|
||||
|
||||
pc = preempt_count();
|
||||
__size = SIZEOF_KPROBE_TRACE_ENTRY(tp->nr_args);
|
||||
|
@ -1229,7 +1229,8 @@ static __kprobes int kprobe_profile_func(struct kprobe *kp,
|
|||
*/
|
||||
local_irq_save(irq_flags);
|
||||
|
||||
if (perf_swevent_get_recursion_context(&recursion))
|
||||
rctx = perf_swevent_get_recursion_context();
|
||||
if (rctx < 0)
|
||||
goto end_recursion;
|
||||
|
||||
__cpu = smp_processor_id();
|
||||
|
@ -1258,7 +1259,7 @@ static __kprobes int kprobe_profile_func(struct kprobe *kp,
|
|||
perf_tp_event(call->id, entry->ip, 1, entry, size);
|
||||
|
||||
end:
|
||||
perf_swevent_put_recursion_context(recursion);
|
||||
perf_swevent_put_recursion_context(rctx);
|
||||
end_recursion:
|
||||
local_irq_restore(irq_flags);
|
||||
|
||||
|
@ -1276,8 +1277,8 @@ static __kprobes int kretprobe_profile_func(struct kretprobe_instance *ri,
|
|||
int size, __size, i, pc, __cpu;
|
||||
unsigned long irq_flags;
|
||||
char *trace_buf;
|
||||
int *recursion;
|
||||
char *raw_data;
|
||||
int rctx;
|
||||
|
||||
pc = preempt_count();
|
||||
__size = SIZEOF_KRETPROBE_TRACE_ENTRY(tp->nr_args);
|
||||
|
@ -1293,7 +1294,8 @@ static __kprobes int kretprobe_profile_func(struct kretprobe_instance *ri,
|
|||
*/
|
||||
local_irq_save(irq_flags);
|
||||
|
||||
if (perf_swevent_get_recursion_context(&recursion))
|
||||
rctx = perf_swevent_get_recursion_context();
|
||||
if (rctx < 0)
|
||||
goto end_recursion;
|
||||
|
||||
__cpu = smp_processor_id();
|
||||
|
@ -1323,7 +1325,7 @@ static __kprobes int kretprobe_profile_func(struct kretprobe_instance *ri,
|
|||
perf_tp_event(call->id, entry->ret_ip, 1, entry, size);
|
||||
|
||||
end:
|
||||
perf_swevent_put_recursion_context(recursion);
|
||||
perf_swevent_put_recursion_context(rctx);
|
||||
end_recursion:
|
||||
local_irq_restore(irq_flags);
|
||||
|
||||
|
|
|
@ -481,8 +481,8 @@ static void prof_syscall_enter(struct pt_regs *regs, long id)
|
|||
unsigned long flags;
|
||||
char *trace_buf;
|
||||
char *raw_data;
|
||||
int *recursion;
|
||||
int syscall_nr;
|
||||
int rctx;
|
||||
int size;
|
||||
int cpu;
|
||||
|
||||
|
@ -506,7 +506,8 @@ static void prof_syscall_enter(struct pt_regs *regs, long id)
|
|||
/* Protect the per cpu buffer, begin the rcu read side */
|
||||
local_irq_save(flags);
|
||||
|
||||
if (perf_swevent_get_recursion_context(&recursion))
|
||||
rctx = perf_swevent_get_recursion_context();
|
||||
if (rctx < 0)
|
||||
goto end_recursion;
|
||||
|
||||
cpu = smp_processor_id();
|
||||
|
@ -530,7 +531,7 @@ static void prof_syscall_enter(struct pt_regs *regs, long id)
|
|||
perf_tp_event(sys_data->enter_id, 0, 1, rec, size);
|
||||
|
||||
end:
|
||||
perf_swevent_put_recursion_context(recursion);
|
||||
perf_swevent_put_recursion_context(rctx);
|
||||
end_recursion:
|
||||
local_irq_restore(flags);
|
||||
}
|
||||
|
@ -582,7 +583,7 @@ static void prof_syscall_exit(struct pt_regs *regs, long ret)
|
|||
int syscall_nr;
|
||||
char *trace_buf;
|
||||
char *raw_data;
|
||||
int *recursion;
|
||||
int rctx;
|
||||
int size;
|
||||
int cpu;
|
||||
|
||||
|
@ -609,7 +610,8 @@ static void prof_syscall_exit(struct pt_regs *regs, long ret)
|
|||
/* Protect the per cpu buffer, begin the rcu read side */
|
||||
local_irq_save(flags);
|
||||
|
||||
if (perf_swevent_get_recursion_context(&recursion))
|
||||
rctx = perf_swevent_get_recursion_context();
|
||||
if (rctx < 0)
|
||||
goto end_recursion;
|
||||
|
||||
cpu = smp_processor_id();
|
||||
|
@ -634,7 +636,7 @@ static void prof_syscall_exit(struct pt_regs *regs, long ret)
|
|||
perf_tp_event(sys_data->exit_id, 0, 1, rec, size);
|
||||
|
||||
end:
|
||||
perf_swevent_put_recursion_context(recursion);
|
||||
perf_swevent_put_recursion_context(rctx);
|
||||
end_recursion:
|
||||
local_irq_restore(flags);
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue